From a9b4bedbcffadeb8d51200a307e1752185afc069 Mon Sep 17 00:00:00 2001 From: Manali Latkar Date: Wed, 6 Nov 2013 15:37:42 +0530 Subject: [PATCH 1/2] stacktach picks up all exists with status sent_unverified and verifies them without sending them to yagi --- stacktach/models.py | 22 +++++++++++-- tests/unit/test_glance_verifier.py | 30 +++++++++++++++-- tests/unit/test_nova_verifier.py | 53 +++++++++++++++++++++++++++--- verifier/glance_verifier.py | 28 +++++++++++----- verifier/nova_verifier.py | 20 ++++++++--- 5 files changed, 131 insertions(+), 22 deletions(-) diff --git a/stacktach/models.py b/stacktach/models.py index b30686e..c1a4c49 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -256,12 +256,18 @@ class InstanceExists(models.Model): VERIFIED = 'verified' RECONCILED = 'reconciled' FAILED = 'failed' + SENT_UNVERIFIED = 'sent_unverified' + SENT_FAILED = 'sent_failed' + SENT_VERIFYING = 'sent_verifying' STATUS_CHOICES = [ (PENDING, 'Pending Verification'), (VERIFYING, 'Currently Being Verified'), (VERIFIED, 'Passed Verification'), (RECONCILED, 'Passed Verification After Reconciliation'), (FAILED, 'Failed Verification'), + (SENT_UNVERIFIED, 'Unverified but sent by Yagi'), + (SENT_FAILED, 'Failed Verification but sent by Yagi'), + (SENT_VERIFYING, 'Currently being verified but sent by Yagi') ] instance = models.CharField(max_length=50, null=True, @@ -321,7 +327,10 @@ class InstanceExists(models.Model): self.save() def mark_failed(self, reason=None): - self.status = InstanceExists.FAILED + if self.status == InstanceExists.SENT_VERIFYING: + self.status = InstanceExists.SENT_FAILED + else: + self.status = InstanceExists.FAILED if reason: self.fail_reason = reason self.save() @@ -458,11 +467,17 @@ class ImageExists(models.Model): VERIFYING = 'verifying' VERIFIED = 'verified' FAILED = 'failed' + SENT_UNVERIFIED = 'sent_unverified' + SENT_FAILED = 'sent_failed' + SENT_VERIFYING = 'sent_verifying' STATUS_CHOICES = [ (PENDING, 'Pending Verification'), (VERIFYING, 'Currently Being Verified'), (VERIFIED, 'Passed Verification'), (FAILED, 'Failed Verification'), + (SENT_UNVERIFIED, 'Unverified but sent by Yagi'), + (SENT_FAILED, 'Failed Verification but sent by Yagi'), + (SENT_VERIFYING, 'Currently being verified but sent by Yagi') ] uuid = models.CharField(max_length=50, db_index=True, null=True) @@ -511,7 +526,10 @@ class ImageExists(models.Model): self.save() def mark_failed(self, reason=None): - self.status = InstanceExists.FAILED + if self.status == ImageExists.SENT_VERIFYING: + self.status = ImageExists.SENT_FAILED + else: + self.status = ImageExists.FAILED if reason: self.fail_reason = reason self.save() diff --git a/tests/unit/test_glance_verifier.py b/tests/unit/test_glance_verifier.py index e783038..9fc7a79 100644 --- a/tests/unit/test_glance_verifier.py +++ b/tests/unit/test_glance_verifier.py @@ -462,25 +462,41 @@ class GlanceVerifierTestCase(StacktachBaseTestCase): self.mox.VerifyAll() self.assertFalse(verified) - def test_verify_for_range_without_callback(self): + + def test_verify_for_range_without_callback_for_sent_unverified(self): mock_logger = self._setup_mock_logger() self.mox.StubOutWithMock(mock_logger, 'info') + stacklog.get_logger('verifier', is_parent=False).AndReturn(mock_logger) + mock_logger.info('glance: Adding 2 per-owner exists to queue.') mock_logger.info('glance: Adding 2 per-owner exists to queue.') - when_max = datetime.utcnow() models.ImageExists.VERIFYING = 'verifying' models.ImageExists.PENDING = 'pending' + models.ImageExists.SENT_VERIFYING = 'sent_verifying' + models.ImageExists.SENT_UNVERIFIED = 'sent_unverified' self.mox.StubOutWithMock(models.ImageExists, 'find') exist1 = self.mox.CreateMockAnything() exist2 = self.mox.CreateMockAnything() exist3 = self.mox.CreateMockAnything() + exist4 = self.mox.CreateMockAnything() + exist5 = self.mox.CreateMockAnything() results = {'owner1': [exist1, exist2], 'owner2': [exist3]} + sent_results = {'owner1': [exist4], 'owner2': [exist5]} + models.ImageExists.find_and_group_by_owner_and_raw_id( + ending_max=when_max, + status=models.ImageExists.SENT_UNVERIFIED).AndReturn(sent_results) models.ImageExists.find_and_group_by_owner_and_raw_id( ending_max=when_max, status=models.ImageExists.PENDING).AndReturn(results) exist1.save() exist2.save() exist3.save() + exist4.save() + exist5.save() + self.pool.apply_async(glance_verifier._verify, + args=([exist4],), callback=None) + self.pool.apply_async(glance_verifier._verify, args=([exist5],), + callback=None) self.pool.apply_async(glance_verifier._verify, args=([exist1, exist2],), callback=None) self.pool.apply_async(glance_verifier._verify, args=([exist3],), @@ -491,21 +507,29 @@ class GlanceVerifierTestCase(StacktachBaseTestCase): self.assertEqual(exist1.status, 'verifying') self.assertEqual(exist2.status, 'verifying') self.assertEqual(exist3.status, 'verifying') + self.assertEqual(exist4.status, 'sent_verifying') + self.assertEqual(exist5.status, 'sent_verifying') self.mox.VerifyAll() def test_verify_for_range_with_callback(self): mock_logger = self._setup_mock_logger() self.mox.StubOutWithMock(mock_logger, 'info') + stacklog.get_logger('verifier', is_parent=False).AndReturn(mock_logger) + mock_logger.info('glance: Adding 0 per-owner exists to queue.') mock_logger.info('glance: Adding 2 per-owner exists to queue.') - callback = self.mox.CreateMockAnything() when_max = datetime.utcnow() + models.ImageExists.SENT_VERIFYING = 'sent_verifying' + models.ImageExists.SENT_UNVERIFIED = 'sent_unverified' models.ImageExists.PENDING = 'pending' models.ImageExists.VERIFYING = 'verifying' exist1 = self.mox.CreateMockAnything() exist2 = self.mox.CreateMockAnything() exist3 = self.mox.CreateMockAnything() results = {'owner1': [exist1, exist2], 'owner2': [exist3]} + models.ImageExists.find_and_group_by_owner_and_raw_id( + ending_max=when_max, + status=models.ImageExists.SENT_UNVERIFIED).AndReturn([]) models.ImageExists.find_and_group_by_owner_and_raw_id( ending_max=when_max, status=models.ImageExists.PENDING).AndReturn(results) diff --git a/tests/unit/test_nova_verifier.py b/tests/unit/test_nova_verifier.py index eb3490d..e56959e 100644 --- a/tests/unit/test_nova_verifier.py +++ b/tests/unit/test_nova_verifier.py @@ -798,18 +798,23 @@ class NovaVerifierVerifyTestCase(StacktachBaseTestCase): self.assertFalse(result) self.mox.VerifyAll() - def test_verify_for_range_without_callback(self): mock_logger = self._create_mock_logger() stacklog.get_logger('verifier', is_parent=False).AndReturn(mock_logger) + stacklog.get_logger('verifier', is_parent=False).AndReturn(mock_logger) + mock_logger.info('nova: Adding 0 exists to queue.') mock_logger.info('nova: Adding 2 exists to queue.') - when_max = datetime.datetime.utcnow() results = self.mox.CreateMockAnything() + sent_results = self.mox.CreateMockAnything() models.InstanceExists.PENDING = 'pending' models.InstanceExists.VERIFYING = 'verifying' + models.InstanceExists.SENT_UNVERIFIED = 'sent_unverified' + models.InstanceExists.find( + ending_max=when_max, status='sent_unverified').AndReturn(sent_results) models.InstanceExists.find( ending_max=when_max, status='pending').AndReturn(results) + sent_results.count().AndReturn(0) results.count().AndReturn(2) exist1 = self.mox.CreateMockAnything() exist2 = self.mox.CreateMockAnything() @@ -827,18 +832,25 @@ class NovaVerifierVerifyTestCase(StacktachBaseTestCase): self.verifier.verify_for_range(when_max) self.mox.VerifyAll() + def test_verify_for_range_with_callback(self): + callback = self.mox.CreateMockAnything() mock_logger = self._create_mock_logger() stacklog.get_logger('verifier', is_parent=False).AndReturn(mock_logger) - mock_logger.info("nova: Adding 2 exists to queue.") - - callback = self.mox.CreateMockAnything() + stacklog.get_logger('verifier', is_parent=False).AndReturn(mock_logger) + mock_logger.info('nova: Adding 0 exists to queue.') + mock_logger.info('nova: Adding 2 exists to queue.') when_max = datetime.datetime.utcnow() results = self.mox.CreateMockAnything() + sent_results = self.mox.CreateMockAnything() models.InstanceExists.PENDING = 'pending' models.InstanceExists.VERIFYING = 'verifying' + models.InstanceExists.SENT_UNVERIFIED = 'sent_unverified' + models.InstanceExists.find( + ending_max=when_max, status='sent_unverified').AndReturn(sent_results) models.InstanceExists.find( ending_max=when_max, status='pending').AndReturn(results) + sent_results.count().AndReturn(0) results.count().AndReturn(2) exist1 = self.mox.CreateMockAnything() exist2 = self.mox.CreateMockAnything() @@ -857,6 +869,37 @@ class NovaVerifierVerifyTestCase(StacktachBaseTestCase): self.mox.VerifyAll() + def test_verify_for_range_when_found_sent_unverified_messages(self): + callback = self.mox.CreateMockAnything() + when_max = datetime.datetime.utcnow() + results = self.mox.CreateMockAnything() + sent_results = self.mox.CreateMockAnything() + models.InstanceExists.PENDING = 'pending' + models.InstanceExists.VERIFYING = 'verifying' + models.InstanceExists.SENT_VERIFYING = 'sent_verifying' + models.InstanceExists.SENT_UNVERIFIED = 'sent_unverified' + models.InstanceExists.find( + ending_max=when_max, status='sent_unverified').AndReturn(sent_results) + models.InstanceExists.find( + ending_max=when_max, status='pending').AndReturn(results) + sent_results.count().AndReturn(2) + results.count().AndReturn(0) + exist1 = self.mox.CreateMockAnything() + exist2 = self.mox.CreateMockAnything() + sent_results.__getslice__(0, 1000).AndReturn(sent_results) + sent_results.__iter__().AndReturn([exist1, exist2].__iter__()) + exist1.update_status('sent_verifying') + exist2.update_status('sent_verifying') + exist1.save() + exist2.save() + self.pool.apply_async(nova_verifier._verify, args=(exist1, 'all'), + callback=None) + self.pool.apply_async(nova_verifier._verify, args=(exist2, 'all'), + callback=None) + self.mox.ReplayAll() + self.verifier.verify_for_range(when_max, callback=callback) + self.mox.VerifyAll() + class NovaVerifierSendVerifiedNotificationTestCase(StacktachBaseTestCase): def setUp(self): self.mox = mox.Mox() diff --git a/verifier/glance_verifier.py b/verifier/glance_verifier.py index 459b0a3..8a8977b 100644 --- a/verifier/glance_verifier.py +++ b/verifier/glance_verifier.py @@ -148,20 +148,16 @@ class GlanceVerifier(Verifier): def __init__(self, config, pool=None): super(GlanceVerifier, self).__init__(config, pool=pool) - def verify_for_range(self, ending_max, callback=None): - exists_grouped_by_owner_and_rawid = \ - models.ImageExists.find_and_group_by_owner_and_raw_id( - ending_max=ending_max, - status=models.ImageExists.PENDING) - count = len(exists_grouped_by_owner_and_rawid) + def verify_exists(self, grouped_exists, callback, verifying_status): + count = len(grouped_exists) added = 0 update_interval = datetime.timedelta(seconds=30) next_update = datetime.datetime.utcnow() + update_interval _get_child_logger().info("glance: Adding %s per-owner exists to queue." % count) while added < count: - for exists in exists_grouped_by_owner_and_rawid.values(): + for exists in grouped_exists.values(): for exist in exists: - exist.status = models.ImageExists.VERIFYING + exist.status = verifying_status exist.save() result = self.pool.apply_async(_verify, args=(exists,), callback=callback) @@ -174,6 +170,22 @@ class GlanceVerifier(Verifier): next_update = datetime.datetime.utcnow() + update_interval return count + def verify_for_range(self, ending_max, callback=None): + unsent_exists_grouped_by_owner_and_rawid = \ + models.ImageExists.find_and_group_by_owner_and_raw_id( + ending_max=ending_max, + status=models.ImageExists.SENT_UNVERIFIED) + unsent_count = self.verify_exists(unsent_exists_grouped_by_owner_and_rawid, + None, models.ImageExists.SENT_VERIFYING) + exists_grouped_by_owner_and_rawid = \ + models.ImageExists.find_and_group_by_owner_and_raw_id( + ending_max=ending_max, + status=models.ImageExists.PENDING) + count = self.verify_exists(exists_grouped_by_owner_and_rawid, callback, + models.ImageExists.VERIFYING) + + return count+unsent_count + def send_verified_notification(self, exist, connection, exchange, routing_keys=None): # NOTE (apmelton) diff --git a/verifier/nova_verifier.py b/verifier/nova_verifier.py index 09165cd..9c91a97 100644 --- a/verifier/nova_verifier.py +++ b/verifier/nova_verifier.py @@ -290,9 +290,7 @@ class NovaVerifier(base_verifier.Verifier): message_service.send_notification( json_body[1], key, connection, exchange) - def verify_for_range(self, ending_max, callback=None): - exists = models.InstanceExists.find( - ending_max=ending_max, status=models.InstanceExists.PENDING) + def verify_exists(self, callback, exists, verifying_status): count = exists.count() added = 0 update_interval = datetime.timedelta(seconds=30) @@ -300,7 +298,7 @@ class NovaVerifier(base_verifier.Verifier): _get_child_logger().info("nova: Adding %s exists to queue." % count) while added < count: for exist in exists[0:1000]: - exist.update_status(models.InstanceExists.VERIFYING) + exist.update_status(verifying_status) exist.save() validation_level = self.config.validation_level() result = self.pool.apply_async( @@ -315,6 +313,20 @@ class NovaVerifier(base_verifier.Verifier): next_update = datetime.datetime.utcnow() + update_interval return count + def verify_for_range(self, ending_max, callback=None): + sent_unverified_exists = models.InstanceExists.find( + ending_max=ending_max, status= + models.InstanceExists.SENT_UNVERIFIED) + sent_unverified_count = self.verify_exists(None, + sent_unverified_exists, + models.InstanceExists. + SENT_VERIFYING) + exists = models.InstanceExists.find( + ending_max=ending_max, status=models.InstanceExists.PENDING) + count = self.verify_exists(callback, exists, + models.InstanceExists.VERIFYING) + return count+sent_unverified_count + def reconcile_failed(self): for failed_exist in self.failed: self.reconciler.failed_validation(failed_exist) From cdff368be0fad404a0eb495e401fa507bd069b48 Mon Sep 17 00:00:00 2001 From: Manali Latkar Date: Thu, 19 Dec 2013 18:31:43 +0530 Subject: [PATCH 2/2] adding the new status counts to usage audit --- reports/glance_usage_audit.py | 2 +- reports/nova_usage_audit.py | 2 +- reports/usage_audit.py | 13 ++++++++++--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/reports/glance_usage_audit.py b/reports/glance_usage_audit.py index 7588d7f..5e50167 100644 --- a/reports/glance_usage_audit.py +++ b/reports/glance_usage_audit.py @@ -176,7 +176,7 @@ def store_results(start, end, summary, details): 'created': dt.dt_to_decimal(datetime.datetime.utcnow()), 'period_start': start, 'period_end': end, - 'version': 4, + 'version': 6, 'name': 'glance usage audit' } diff --git a/reports/nova_usage_audit.py b/reports/nova_usage_audit.py index 5afebbd..b4ed71c 100644 --- a/reports/nova_usage_audit.py +++ b/reports/nova_usage_audit.py @@ -224,7 +224,7 @@ def store_results(start, end, summary, details): 'created': dt.dt_to_decimal(datetime.datetime.utcnow()), 'period_start': start, 'period_end': end, - 'version': 5, + 'version': 6, 'name': 'nova usage audit' } diff --git a/reports/usage_audit.py b/reports/usage_audit.py index 284c57d..4d62ac4 100644 --- a/reports/usage_audit.py +++ b/reports/usage_audit.py @@ -9,8 +9,11 @@ def _status_queries(exists_query): fail = exists_query.filter(status=models.InstanceExists.FAILED) pending = exists_query.filter(status=models.InstanceExists.PENDING) verifying = exists_query.filter(status=models.InstanceExists.VERIFYING) - - return verified, reconciled, fail, pending, verifying + sent_unverified = exists_query.filter(status=models.InstanceExists.SENT_UNVERIFIED) + sent_failed = exists_query.filter(status=models.InstanceExists.VERIFYING) + sent_verifying = exists_query.filter(status=models.InstanceExists.SENT_VERIFYING) + return verified, reconciled, fail, pending, verifying, sent_unverified, \ + sent_failed, sent_verifying def _send_status_queries(exists_query): @@ -28,7 +31,8 @@ def _send_status_queries(exists_query): def _audit_for_exists(exists_query): (verified, reconciled, - fail, pending, verifying) = _status_queries(exists_query) + fail, pending, verifying, sent_unverified, + sent_failed, sent_verifying) = _status_queries(exists_query) (success, unsent, redirect, client_error, server_error) = _send_status_queries(verified) @@ -43,6 +47,9 @@ def _audit_for_exists(exists_query): 'failed': fail.count(), 'pending': pending.count(), 'verifying': verifying.count(), + 'sent_unverified': sent_unverified.count(), + 'sent_failed': sent_failed.count(), + 'sent_verifying': sent_verifying.count(), 'send_status': { 'success': success.count(), 'unsent': unsent.count(),