stacktach picks up all exists with status sent_unverified and verifies them without sending them to yagi

stacktach/models.py

@@ -256,12 +256,18 @@ class InstanceExists(models.Model):
     VERIFIED = 'verified'
     RECONCILED = 'reconciled'
     FAILED = 'failed'
+    SENT_UNVERIFIED = 'sent_unverified'
+    SENT_FAILED = 'sent_failed'
+    SENT_VERIFYING = 'sent_verifying'
     STATUS_CHOICES = [
         (PENDING, 'Pending Verification'),
         (VERIFYING, 'Currently Being Verified'),
         (VERIFIED, 'Passed Verification'),
         (RECONCILED, 'Passed Verification After Reconciliation'),
         (FAILED, 'Failed Verification'),
+        (SENT_UNVERIFIED, 'Unverified but sent by Yagi'),
+        (SENT_FAILED, 'Failed Verification but sent by Yagi'),
+        (SENT_VERIFYING, 'Currently being verified but sent by Yagi')
     ]
 
     instance = models.CharField(max_length=50, null=True,
@@ -321,6 +327,9 @@ class InstanceExists(models.Model):
         self.save()
 
     def mark_failed(self, reason=None):
+        if self.status == InstanceExists.SENT_VERIFYING:
+            self.status = InstanceExists.SENT_FAILED
+        else:
             self.status = InstanceExists.FAILED
         if reason:
             self.fail_reason = reason
@@ -458,11 +467,17 @@ class ImageExists(models.Model):
     VERIFYING = 'verifying'
     VERIFIED = 'verified'
     FAILED = 'failed'
+    SENT_UNVERIFIED = 'sent_unverified'
+    SENT_FAILED = 'sent_failed'
+    SENT_VERIFYING = 'sent_verifying'
     STATUS_CHOICES = [
         (PENDING, 'Pending Verification'),
         (VERIFYING, 'Currently Being Verified'),
         (VERIFIED, 'Passed Verification'),
         (FAILED, 'Failed Verification'),
+        (SENT_UNVERIFIED, 'Unverified but sent by Yagi'),
+        (SENT_FAILED, 'Failed Verification but sent by Yagi'),
+        (SENT_VERIFYING, 'Currently being verified but sent by Yagi')
     ]
 
     uuid = models.CharField(max_length=50, db_index=True, null=True)
@@ -511,7 +526,10 @@ class ImageExists(models.Model):
         self.save()
 
     def mark_failed(self, reason=None):
-        self.status = InstanceExists.FAILED
+        if self.status == ImageExists.SENT_VERIFYING:
+            self.status = ImageExists.SENT_FAILED
+        else:
+            self.status = ImageExists.FAILED
         if reason:
             self.fail_reason = reason
         self.save()

tests/unit/test_glance_verifier.py

@@ -462,25 +462,41 @@ class GlanceVerifierTestCase(StacktachBaseTestCase):
         self.mox.VerifyAll()
         self.assertFalse(verified)
 
-    def test_verify_for_range_without_callback(self):
+
+    def test_verify_for_range_without_callback_for_sent_unverified(self):
         mock_logger = self._setup_mock_logger()
         self.mox.StubOutWithMock(mock_logger, 'info')
+        stacklog.get_logger('verifier', is_parent=False).AndReturn(mock_logger)
+        mock_logger.info('glance: Adding 2 per-owner exists to queue.')
         mock_logger.info('glance: Adding 2 per-owner exists to queue.')
-
         when_max = datetime.utcnow()
         models.ImageExists.VERIFYING = 'verifying'
         models.ImageExists.PENDING = 'pending'
+        models.ImageExists.SENT_VERIFYING = 'sent_verifying'
+        models.ImageExists.SENT_UNVERIFIED = 'sent_unverified'
         self.mox.StubOutWithMock(models.ImageExists, 'find')
         exist1 = self.mox.CreateMockAnything()
         exist2 = self.mox.CreateMockAnything()
         exist3 = self.mox.CreateMockAnything()
+        exist4 = self.mox.CreateMockAnything()
+        exist5 = self.mox.CreateMockAnything()
         results = {'owner1': [exist1, exist2], 'owner2': [exist3]}
+        sent_results = {'owner1': [exist4], 'owner2': [exist5]}
+        models.ImageExists.find_and_group_by_owner_and_raw_id(
+            ending_max=when_max,
+            status=models.ImageExists.SENT_UNVERIFIED).AndReturn(sent_results)
         models.ImageExists.find_and_group_by_owner_and_raw_id(
             ending_max=when_max,
             status=models.ImageExists.PENDING).AndReturn(results)
         exist1.save()
         exist2.save()
         exist3.save()
+        exist4.save()
+        exist5.save()
+        self.pool.apply_async(glance_verifier._verify,
+                              args=([exist4],), callback=None)
+        self.pool.apply_async(glance_verifier._verify, args=([exist5],),
+                              callback=None)
         self.pool.apply_async(glance_verifier._verify,
                               args=([exist1, exist2],), callback=None)
         self.pool.apply_async(glance_verifier._verify, args=([exist3],),
@@ -491,21 +507,29 @@ class GlanceVerifierTestCase(StacktachBaseTestCase):
         self.assertEqual(exist1.status, 'verifying')
         self.assertEqual(exist2.status, 'verifying')
         self.assertEqual(exist3.status, 'verifying')
+        self.assertEqual(exist4.status, 'sent_verifying')
+        self.assertEqual(exist5.status, 'sent_verifying')
         self.mox.VerifyAll()
 
     def test_verify_for_range_with_callback(self):
         mock_logger = self._setup_mock_logger()
         self.mox.StubOutWithMock(mock_logger, 'info')
+        stacklog.get_logger('verifier', is_parent=False).AndReturn(mock_logger)
+        mock_logger.info('glance: Adding 0 per-owner exists to queue.')
         mock_logger.info('glance: Adding 2 per-owner exists to queue.')
-
         callback = self.mox.CreateMockAnything()
         when_max = datetime.utcnow()
+        models.ImageExists.SENT_VERIFYING = 'sent_verifying'
+        models.ImageExists.SENT_UNVERIFIED = 'sent_unverified'
         models.ImageExists.PENDING = 'pending'
         models.ImageExists.VERIFYING = 'verifying'
         exist1 = self.mox.CreateMockAnything()
         exist2 = self.mox.CreateMockAnything()
         exist3 = self.mox.CreateMockAnything()
         results = {'owner1': [exist1, exist2], 'owner2': [exist3]}
+        models.ImageExists.find_and_group_by_owner_and_raw_id(
+            ending_max=when_max,
+            status=models.ImageExists.SENT_UNVERIFIED).AndReturn([])
         models.ImageExists.find_and_group_by_owner_and_raw_id(
             ending_max=when_max,
             status=models.ImageExists.PENDING).AndReturn(results)

tests/unit/test_nova_verifier.py

@@ -798,18 +798,23 @@ class NovaVerifierVerifyTestCase(StacktachBaseTestCase):
         self.assertFalse(result)
         self.mox.VerifyAll()
 
-
     def test_verify_for_range_without_callback(self):
         mock_logger = self._create_mock_logger()
         stacklog.get_logger('verifier', is_parent=False).AndReturn(mock_logger)
+        stacklog.get_logger('verifier', is_parent=False).AndReturn(mock_logger)
+        mock_logger.info('nova: Adding 0 exists to queue.')
         mock_logger.info('nova: Adding 2 exists to queue.')
-
         when_max = datetime.datetime.utcnow()
         results = self.mox.CreateMockAnything()
+        sent_results = self.mox.CreateMockAnything()
         models.InstanceExists.PENDING = 'pending'
         models.InstanceExists.VERIFYING = 'verifying'
+        models.InstanceExists.SENT_UNVERIFIED = 'sent_unverified'
+        models.InstanceExists.find(
+            ending_max=when_max, status='sent_unverified').AndReturn(sent_results)
         models.InstanceExists.find(
             ending_max=when_max, status='pending').AndReturn(results)
+        sent_results.count().AndReturn(0)
         results.count().AndReturn(2)
         exist1 = self.mox.CreateMockAnything()
         exist2 = self.mox.CreateMockAnything()
@@ -827,18 +832,25 @@ class NovaVerifierVerifyTestCase(StacktachBaseTestCase):
         self.verifier.verify_for_range(when_max)
         self.mox.VerifyAll()
 
+
     def test_verify_for_range_with_callback(self):
+        callback = self.mox.CreateMockAnything()
         mock_logger = self._create_mock_logger()
         stacklog.get_logger('verifier', is_parent=False).AndReturn(mock_logger)
-        mock_logger.info("nova: Adding 2 exists to queue.")
+        stacklog.get_logger('verifier', is_parent=False).AndReturn(mock_logger)
-
+        mock_logger.info('nova: Adding 0 exists to queue.')
-        callback = self.mox.CreateMockAnything()
+        mock_logger.info('nova: Adding 2 exists to queue.')
         when_max = datetime.datetime.utcnow()
         results = self.mox.CreateMockAnything()
+        sent_results = self.mox.CreateMockAnything()
         models.InstanceExists.PENDING = 'pending'
         models.InstanceExists.VERIFYING = 'verifying'
+        models.InstanceExists.SENT_UNVERIFIED = 'sent_unverified'
+        models.InstanceExists.find(
+            ending_max=when_max, status='sent_unverified').AndReturn(sent_results)
         models.InstanceExists.find(
             ending_max=when_max, status='pending').AndReturn(results)
+        sent_results.count().AndReturn(0)
         results.count().AndReturn(2)
         exist1 = self.mox.CreateMockAnything()
         exist2 = self.mox.CreateMockAnything()
@@ -857,6 +869,37 @@ class NovaVerifierVerifyTestCase(StacktachBaseTestCase):
         self.mox.VerifyAll()
 
 
+    def test_verify_for_range_when_found_sent_unverified_messages(self):
+        callback = self.mox.CreateMockAnything()
+        when_max = datetime.datetime.utcnow()
+        results = self.mox.CreateMockAnything()
+        sent_results = self.mox.CreateMockAnything()
+        models.InstanceExists.PENDING = 'pending'
+        models.InstanceExists.VERIFYING = 'verifying'
+        models.InstanceExists.SENT_VERIFYING = 'sent_verifying'
+        models.InstanceExists.SENT_UNVERIFIED = 'sent_unverified'
+        models.InstanceExists.find(
+            ending_max=when_max, status='sent_unverified').AndReturn(sent_results)
+        models.InstanceExists.find(
+            ending_max=when_max, status='pending').AndReturn(results)
+        sent_results.count().AndReturn(2)
+        results.count().AndReturn(0)
+        exist1 = self.mox.CreateMockAnything()
+        exist2 = self.mox.CreateMockAnything()
+        sent_results.__getslice__(0, 1000).AndReturn(sent_results)
+        sent_results.__iter__().AndReturn([exist1, exist2].__iter__())
+        exist1.update_status('sent_verifying')
+        exist2.update_status('sent_verifying')
+        exist1.save()
+        exist2.save()
+        self.pool.apply_async(nova_verifier._verify, args=(exist1, 'all'),
+                              callback=None)
+        self.pool.apply_async(nova_verifier._verify, args=(exist2, 'all'),
+                              callback=None)
+        self.mox.ReplayAll()
+        self.verifier.verify_for_range(when_max, callback=callback)
+        self.mox.VerifyAll()
+
 class NovaVerifierSendVerifiedNotificationTestCase(StacktachBaseTestCase):
     def setUp(self):
         self.mox = mox.Mox()

verifier/glance_verifier.py

@@ -148,20 +148,16 @@ class GlanceVerifier(Verifier):
     def __init__(self, config, pool=None):
         super(GlanceVerifier, self).__init__(config, pool=pool)
 
-    def verify_for_range(self, ending_max, callback=None):
+    def verify_exists(self, grouped_exists, callback, verifying_status):
-        exists_grouped_by_owner_and_rawid = \
+        count = len(grouped_exists)
-            models.ImageExists.find_and_group_by_owner_and_raw_id(
-                ending_max=ending_max,
-                status=models.ImageExists.PENDING)
-        count = len(exists_grouped_by_owner_and_rawid)
         added = 0
         update_interval = datetime.timedelta(seconds=30)
         next_update = datetime.datetime.utcnow() + update_interval
         _get_child_logger().info("glance: Adding %s per-owner exists to queue." % count)
         while added < count:
-            for exists in exists_grouped_by_owner_and_rawid.values():
+            for exists in grouped_exists.values():
                 for exist in exists:
-                    exist.status = models.ImageExists.VERIFYING
+                    exist.status = verifying_status
                     exist.save()
                 result = self.pool.apply_async(_verify, args=(exists,),
                                                callback=callback)
@@ -174,6 +170,22 @@ class GlanceVerifier(Verifier):
                     next_update = datetime.datetime.utcnow() + update_interval
         return count
 
+    def verify_for_range(self, ending_max, callback=None):
+        unsent_exists_grouped_by_owner_and_rawid = \
+            models.ImageExists.find_and_group_by_owner_and_raw_id(
+                ending_max=ending_max,
+                status=models.ImageExists.SENT_UNVERIFIED)
+        unsent_count = self.verify_exists(unsent_exists_grouped_by_owner_and_rawid,
+                                        None, models.ImageExists.SENT_VERIFYING)
+        exists_grouped_by_owner_and_rawid = \
+            models.ImageExists.find_and_group_by_owner_and_raw_id(
+                ending_max=ending_max,
+                status=models.ImageExists.PENDING)
+        count = self.verify_exists(exists_grouped_by_owner_and_rawid, callback,
+                                 models.ImageExists.VERIFYING)
+
+        return count+unsent_count
+
     def send_verified_notification(self, exist, connection, exchange,
                                    routing_keys=None):
         # NOTE (apmelton)

verifier/nova_verifier.py

@@ -290,9 +290,7 @@ class NovaVerifier(base_verifier.Verifier):
                 message_service.send_notification(
                     json_body[1], key, connection, exchange)
 
-    def verify_for_range(self, ending_max, callback=None):
+    def verify_exists(self, callback, exists, verifying_status):
-        exists = models.InstanceExists.find(
-            ending_max=ending_max, status=models.InstanceExists.PENDING)
         count = exists.count()
         added = 0
         update_interval = datetime.timedelta(seconds=30)
@@ -300,7 +298,7 @@ class NovaVerifier(base_verifier.Verifier):
         _get_child_logger().info("nova: Adding %s exists to queue." % count)
         while added < count:
             for exist in exists[0:1000]:
-                exist.update_status(models.InstanceExists.VERIFYING)
+                exist.update_status(verifying_status)
                 exist.save()
                 validation_level = self.config.validation_level()
                 result = self.pool.apply_async(
@@ -315,6 +313,20 @@ class NovaVerifier(base_verifier.Verifier):
                     next_update = datetime.datetime.utcnow() + update_interval
         return count
 
+    def verify_for_range(self, ending_max, callback=None):
+        sent_unverified_exists = models.InstanceExists.find(
+            ending_max=ending_max, status=
+            models.InstanceExists.SENT_UNVERIFIED)
+        sent_unverified_count = self.verify_exists(None,
+                                                   sent_unverified_exists,
+                                                   models.InstanceExists.
+                                                   SENT_VERIFYING)
+        exists = models.InstanceExists.find(
+            ending_max=ending_max, status=models.InstanceExists.PENDING)
+        count = self.verify_exists(callback, exists,
+                                   models.InstanceExists.VERIFYING)
+        return count+sent_unverified_count
+
     def reconcile_failed(self):
         for failed_exist in self.failed:
             self.reconciler.failed_validation(failed_exist)