x/stacktach
Code Issues Proposed changes

Merge pull request #251 from manalilatkar/stacktach_down_scenario

Browse Source 
stacktach picks up all exists with status sent_unverified and verifies them without sending them to yagi
This commit is contained in:
Andrew Melton 2014-01-06 07:48:11 -08:00
commit 0b1874fc05
8 changed files with 143 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
reports/glance_usage_audit.py
View File

@ -176,7 +176,7 @@ def store_results(start, end, summary, details):
'created': dt.dt_to_decimal(datetime.datetime.utcnow()),
'period_start': start,
'period_end': end,
'version': 4,
'version': 6,
'name': 'glance usage audit'
}

2
reports/nova_usage_audit.py
View File

@ -224,7 +224,7 @@ def store_results(start, end, summary, details):
'created': dt.dt_to_decimal(datetime.datetime.utcnow()),
'period_start': start,
'period_end': end,
'version': 5,
'version': 6,
'name': 'nova usage audit'
}

13
reports/usage_audit.py
View File

@ -9,8 +9,11 @@ def _status_queries(exists_query):
fail = exists_query.filter(status=models.InstanceExists.FAILED)
pending = exists_query.filter(status=models.InstanceExists.PENDING)
verifying = exists_query.filter(status=models.InstanceExists.VERIFYING)
return verified, reconciled, fail, pending, verifying
sent_unverified = exists_query.filter(status=models.InstanceExists.SENT_UNVERIFIED)
sent_failed = exists_query.filter(status=models.InstanceExists.VERIFYING)
sent_verifying = exists_query.filter(status=models.InstanceExists.SENT_VERIFYING)
return verified, reconciled, fail, pending, verifying, sent_unverified, \
sent_failed, sent_verifying
def _send_status_queries(exists_query):
@ -28,7 +31,8 @@ def _send_status_queries(exists_query):
def _audit_for_exists(exists_query):
(verified, reconciled,
fail, pending, verifying) = _status_queries(exists_query)
fail, pending, verifying, sent_unverified,
sent_failed, sent_verifying) = _status_queries(exists_query)
(success, unsent, redirect,
client_error, server_error) = _send_status_queries(verified)
@ -43,6 +47,9 @@ def _audit_for_exists(exists_query):
'failed': fail.count(),
'pending': pending.count(),
'verifying': verifying.count(),
'sent_unverified': sent_unverified.count(),
'sent_failed': sent_failed.count(),
'sent_verifying': sent_verifying.count(),
'send_status': {
'success': success.count(),
'unsent': unsent.count(),

22
stacktach/models.py
View File

@ -256,12 +256,18 @@ class InstanceExists(models.Model):
VERIFIED = 'verified'
RECONCILED = 'reconciled'
FAILED = 'failed'
SENT_UNVERIFIED = 'sent_unverified'
SENT_FAILED = 'sent_failed'
SENT_VERIFYING = 'sent_verifying'
STATUS_CHOICES = [
(PENDING, 'Pending Verification'),
(VERIFYING, 'Currently Being Verified'),
(VERIFIED, 'Passed Verification'),
(RECONCILED, 'Passed Verification After Reconciliation'),
(FAILED, 'Failed Verification'),
(SENT_UNVERIFIED, 'Unverified but sent by Yagi'),
(SENT_FAILED, 'Failed Verification but sent by Yagi'),
(SENT_VERIFYING, 'Currently being verified but sent by Yagi')
]
instance = models.CharField(max_length=50, null=True,
@ -321,7 +327,10 @@ class InstanceExists(models.Model):
self.save()
def mark_failed(self, reason=None):
self.status = InstanceExists.FAILED
if self.status == InstanceExists.SENT_VERIFYING:
self.status = InstanceExists.SENT_FAILED
else:
self.status = InstanceExists.FAILED
if reason:
self.fail_reason = reason
self.save()
@ -458,11 +467,17 @@ class ImageExists(models.Model):
VERIFYING = 'verifying'
VERIFIED = 'verified'
FAILED = 'failed'
SENT_UNVERIFIED = 'sent_unverified'
SENT_FAILED = 'sent_failed'
SENT_VERIFYING = 'sent_verifying'
STATUS_CHOICES = [
(PENDING, 'Pending Verification'),
(VERIFYING, 'Currently Being Verified'),
(VERIFIED, 'Passed Verification'),
(FAILED, 'Failed Verification'),
(SENT_UNVERIFIED, 'Unverified but sent by Yagi'),
(SENT_FAILED, 'Failed Verification but sent by Yagi'),
(SENT_VERIFYING, 'Currently being verified but sent by Yagi')
]
uuid = models.CharField(max_length=50, db_index=True, null=True)
@ -513,7 +528,10 @@ class ImageExists(models.Model):
self.save()
def mark_failed(self, reason=None):
self.status = InstanceExists.FAILED
if self.status == ImageExists.SENT_VERIFYING:
self.status = ImageExists.SENT_FAILED
else:
self.status = ImageExists.FAILED
if reason:
self.fail_reason = reason
self.save()

30
tests/unit/test_glance_verifier.py
View File

@ -458,25 +458,41 @@ class GlanceVerifierTestCase(StacktachBaseTestCase):
self.mox.VerifyAll()
self.assertFalse(verified)
def test_verify_for_range_without_callback(self):
def test_verify_for_range_without_callback_for_sent_unverified(self):
mock_logger = self._setup_mock_logger()
self.mox.StubOutWithMock(mock_logger, 'info')
stacklog.get_logger('verifier', is_parent=False).AndReturn(mock_logger)
mock_logger.info('glance: Adding 2 per-owner exists to queue.')
mock_logger.info('glance: Adding 2 per-owner exists to queue.')
when_max = datetime.utcnow()
models.ImageExists.VERIFYING = 'verifying'
models.ImageExists.PENDING = 'pending'
models.ImageExists.SENT_VERIFYING = 'sent_verifying'
models.ImageExists.SENT_UNVERIFIED = 'sent_unverified'
self.mox.StubOutWithMock(models.ImageExists, 'find')
exist1 = self.mox.CreateMockAnything()
exist2 = self.mox.CreateMockAnything()
exist3 = self.mox.CreateMockAnything()
exist4 = self.mox.CreateMockAnything()
exist5 = self.mox.CreateMockAnything()
results = {'owner1': [exist1, exist2], 'owner2': [exist3]}
sent_results = {'owner1': [exist4], 'owner2': [exist5]}
models.ImageExists.find_and_group_by_owner_and_raw_id(
ending_max=when_max,
status=models.ImageExists.SENT_UNVERIFIED).AndReturn(sent_results)
models.ImageExists.find_and_group_by_owner_and_raw_id(
ending_max=when_max,
status=models.ImageExists.PENDING).AndReturn(results)
exist1.save()
exist2.save()
exist3.save()
exist4.save()
exist5.save()
self.pool.apply_async(glance_verifier._verify,
args=([exist4],), callback=None)
self.pool.apply_async(glance_verifier._verify, args=([exist5],),
callback=None)
self.pool.apply_async(glance_verifier._verify,
args=([exist1, exist2],), callback=None)
self.pool.apply_async(glance_verifier._verify, args=([exist3],),
@ -487,21 +503,29 @@ class GlanceVerifierTestCase(StacktachBaseTestCase):
self.assertEqual(exist1.status, 'verifying')
self.assertEqual(exist2.status, 'verifying')
self.assertEqual(exist3.status, 'verifying')
self.assertEqual(exist4.status, 'sent_verifying')
self.assertEqual(exist5.status, 'sent_verifying')
self.mox.VerifyAll()
def test_verify_for_range_with_callback(self):
mock_logger = self._setup_mock_logger()
self.mox.StubOutWithMock(mock_logger, 'info')
stacklog.get_logger('verifier', is_parent=False).AndReturn(mock_logger)
mock_logger.info('glance: Adding 0 per-owner exists to queue.')
mock_logger.info('glance: Adding 2 per-owner exists to queue.')
callback = self.mox.CreateMockAnything()
when_max = datetime.utcnow()
models.ImageExists.SENT_VERIFYING = 'sent_verifying'
models.ImageExists.SENT_UNVERIFIED = 'sent_unverified'
models.ImageExists.PENDING = 'pending'
models.ImageExists.VERIFYING = 'verifying'
exist1 = self.mox.CreateMockAnything()
exist2 = self.mox.CreateMockAnything()
exist3 = self.mox.CreateMockAnything()
results = {'owner1': [exist1, exist2], 'owner2': [exist3]}
models.ImageExists.find_and_group_by_owner_and_raw_id(
ending_max=when_max,
status=models.ImageExists.SENT_UNVERIFIED).AndReturn([])
models.ImageExists.find_and_group_by_owner_and_raw_id(
ending_max=when_max,
status=models.ImageExists.PENDING).AndReturn(results)

53
tests/unit/test_nova_verifier.py
View File

@ -798,18 +798,23 @@ class NovaVerifierVerifyTestCase(StacktachBaseTestCase):
self.assertFalse(result)
self.mox.VerifyAll()
def test_verify_for_range_without_callback(self):
mock_logger = self._create_mock_logger()
stacklog.get_logger('verifier', is_parent=False).AndReturn(mock_logger)
stacklog.get_logger('verifier', is_parent=False).AndReturn(mock_logger)
mock_logger.info('nova: Adding 0 exists to queue.')
mock_logger.info('nova: Adding 2 exists to queue.')
when_max = datetime.datetime.utcnow()
results = self.mox.CreateMockAnything()
sent_results = self.mox.CreateMockAnything()
models.InstanceExists.PENDING = 'pending'
models.InstanceExists.VERIFYING = 'verifying'
models.InstanceExists.SENT_UNVERIFIED = 'sent_unverified'
models.InstanceExists.find(
ending_max=when_max, status='sent_unverified').AndReturn(sent_results)
models.InstanceExists.find(
ending_max=when_max, status='pending').AndReturn(results)
sent_results.count().AndReturn(0)
results.count().AndReturn(2)
exist1 = self.mox.CreateMockAnything()
exist2 = self.mox.CreateMockAnything()
@ -827,18 +832,25 @@ class NovaVerifierVerifyTestCase(StacktachBaseTestCase):
self.verifier.verify_for_range(when_max)
self.mox.VerifyAll()
def test_verify_for_range_with_callback(self):
callback = self.mox.CreateMockAnything()
mock_logger = self._create_mock_logger()
stacklog.get_logger('verifier', is_parent=False).AndReturn(mock_logger)
mock_logger.info("nova: Adding 2 exists to queue.")
callback = self.mox.CreateMockAnything()
stacklog.get_logger('verifier', is_parent=False).AndReturn(mock_logger)
mock_logger.info('nova: Adding 0 exists to queue.')
mock_logger.info('nova: Adding 2 exists to queue.')
when_max = datetime.datetime.utcnow()
results = self.mox.CreateMockAnything()
sent_results = self.mox.CreateMockAnything()
models.InstanceExists.PENDING = 'pending'
models.InstanceExists.VERIFYING = 'verifying'
models.InstanceExists.SENT_UNVERIFIED = 'sent_unverified'
models.InstanceExists.find(
ending_max=when_max, status='sent_unverified').AndReturn(sent_results)
models.InstanceExists.find(
ending_max=when_max, status='pending').AndReturn(results)
sent_results.count().AndReturn(0)
results.count().AndReturn(2)
exist1 = self.mox.CreateMockAnything()
exist2 = self.mox.CreateMockAnything()
@ -857,6 +869,37 @@ class NovaVerifierVerifyTestCase(StacktachBaseTestCase):
self.mox.VerifyAll()
def test_verify_for_range_when_found_sent_unverified_messages(self):
callback = self.mox.CreateMockAnything()
when_max = datetime.datetime.utcnow()
results = self.mox.CreateMockAnything()
sent_results = self.mox.CreateMockAnything()
models.InstanceExists.PENDING = 'pending'
models.InstanceExists.VERIFYING = 'verifying'
models.InstanceExists.SENT_VERIFYING = 'sent_verifying'
models.InstanceExists.SENT_UNVERIFIED = 'sent_unverified'
models.InstanceExists.find(
ending_max=when_max, status='sent_unverified').AndReturn(sent_results)
models.InstanceExists.find(
ending_max=when_max, status='pending').AndReturn(results)
sent_results.count().AndReturn(2)
results.count().AndReturn(0)
exist1 = self.mox.CreateMockAnything()
exist2 = self.mox.CreateMockAnything()
sent_results.__getslice__(0, 1000).AndReturn(sent_results)
sent_results.__iter__().AndReturn([exist1, exist2].__iter__())
exist1.update_status('sent_verifying')
exist2.update_status('sent_verifying')
exist1.save()
exist2.save()
self.pool.apply_async(nova_verifier._verify, args=(exist1, 'all'),
callback=None)
self.pool.apply_async(nova_verifier._verify, args=(exist2, 'all'),
callback=None)
self.mox.ReplayAll()
self.verifier.verify_for_range(when_max, callback=callback)
self.mox.VerifyAll()
class NovaVerifierSendVerifiedNotificationTestCase(StacktachBaseTestCase):
def setUp(self):
self.mox = mox.Mox()

28
verifier/glance_verifier.py
View File

@ -151,20 +151,16 @@ class GlanceVerifier(Verifier):
def __init__(self, config, pool=None):
super(GlanceVerifier, self).__init__(config, pool=pool)
def verify_for_range(self, ending_max, callback=None):
exists_grouped_by_owner_and_rawid = \
models.ImageExists.find_and_group_by_owner_and_raw_id(
ending_max=ending_max,
status=models.ImageExists.PENDING)
count = len(exists_grouped_by_owner_and_rawid)
def verify_exists(self, grouped_exists, callback, verifying_status):
count = len(grouped_exists)
added = 0
update_interval = datetime.timedelta(seconds=30)
next_update = datetime.datetime.utcnow() + update_interval
_get_child_logger().info("glance: Adding %s per-owner exists to queue." % count)
while added < count:
for exists in exists_grouped_by_owner_and_rawid.values():
for exists in grouped_exists.values():
for exist in exists:
exist.status = models.ImageExists.VERIFYING
exist.status = verifying_status
exist.save()
result = self.pool.apply_async(_verify, args=(exists,),
callback=callback)
@ -177,6 +173,22 @@ class GlanceVerifier(Verifier):
next_update = datetime.datetime.utcnow() + update_interval
return count
def verify_for_range(self, ending_max, callback=None):
unsent_exists_grouped_by_owner_and_rawid = \
models.ImageExists.find_and_group_by_owner_and_raw_id(
ending_max=ending_max,
status=models.ImageExists.SENT_UNVERIFIED)
unsent_count = self.verify_exists(unsent_exists_grouped_by_owner_and_rawid,
None, models.ImageExists.SENT_VERIFYING)
exists_grouped_by_owner_and_rawid = \
models.ImageExists.find_and_group_by_owner_and_raw_id(
ending_max=ending_max,
status=models.ImageExists.PENDING)
count = self.verify_exists(exists_grouped_by_owner_and_rawid, callback,
models.ImageExists.VERIFYING)
return count+unsent_count
def send_verified_notification(self, exist, connection, exchange,
routing_keys=None):
# NOTE (apmelton)

20
verifier/nova_verifier.py
View File

@ -290,9 +290,7 @@ class NovaVerifier(base_verifier.Verifier):
message_service.send_notification(
json_body[1], key, connection, exchange)
def verify_for_range(self, ending_max, callback=None):
exists = models.InstanceExists.find(
ending_max=ending_max, status=models.InstanceExists.PENDING)
def verify_exists(self, callback, exists, verifying_status):
count = exists.count()
added = 0
update_interval = datetime.timedelta(seconds=30)
@ -300,7 +298,7 @@ class NovaVerifier(base_verifier.Verifier):
_get_child_logger().info("nova: Adding %s exists to queue." % count)
while added < count:
for exist in exists[0:1000]:
exist.update_status(models.InstanceExists.VERIFYING)
exist.update_status(verifying_status)
exist.save()
validation_level = self.config.validation_level()
result = self.pool.apply_async(
@ -315,6 +313,20 @@ class NovaVerifier(base_verifier.Verifier):
next_update = datetime.datetime.utcnow() + update_interval
return count
def verify_for_range(self, ending_max, callback=None):
sent_unverified_exists = models.InstanceExists.find(
ending_max=ending_max, status=
models.InstanceExists.SENT_UNVERIFIED)
sent_unverified_count = self.verify_exists(None,
sent_unverified_exists,
models.InstanceExists.
SENT_VERIFYING)
exists = models.InstanceExists.find(
ending_max=ending_max, status=models.InstanceExists.PENDING)
count = self.verify_exists(callback, exists,
models.InstanceExists.VERIFYING)
return count+sent_unverified_count
def reconcile_failed(self):
for failed_exist in self.failed:
self.reconciler.failed_validation(failed_exist)