Merge pull request #135 from ramielrowe/reconciler

Reconciliation in Nova Usage Audit

etc/sample_reconciler_config.json

@@ -0,0 +1,13 @@
+{
+    "client_class": "JSONBridgeClient",
+    "client": {
+        "url": "http://jsonbridge.example.com:8080/query/",
+        "username": "bridgeuser",
+        "password": "super_secure_password",
+        "regions": {
+            "RegionOne": "nova-regionone",
+            "RegionTwo": "nova-regiontwo"
+        }
+    },
+    "region_mapping_loc": "etc/sample_region_mapping.json"
+}

etc/sample_region_mapping.json

@@ -0,0 +1,6 @@
+{
+    "RegionOne.dev.global": "RegionOne",
+    "RegionOne.dev.cell1": "RegionOne",
+    "RegionTwo.dev.global": "RegionTwo",
+    "RegionTwo.dev.cell1": "RegionTwo"
+}

reports/nova_usage_audit.py

@@ -30,12 +30,21 @@ from django.db.models import F
 
 from stacktach import datetime_to_decimal as dt
 from stacktach import models
+from stacktach.reconciler import Reconciler
 
-OLD_LAUNCHES_QUERY = "select * from stacktach_instanceusage " \
+OLD_LAUNCHES_QUERY = """
-                     "where launched_at is not null and " \
+select * from stacktach_instanceusage where
-                     "launched_at < %s and instance not in " \
+    launched_at is not null and
-                     "(select distinct(instance) " \
+    launched_at < %s and
-                     "from stacktach_instancedeletes where deleted_at < %s)"
+    instance not in
+        (select distinct(instance)
+            from stacktach_instancedeletes where
+                deleted_at < %s union
+        select distinct(instance)
+            from stacktach_instancereconcile where
+                deleted_at < %s);"""
+
+reconciler = None
 
 
 def _get_new_launches(beginning, ending):
@@ -63,25 +72,34 @@ def _get_exists(beginning, ending):
     return models.InstanceExists.objects.filter(**filters)
 
 
-def _audit_launches_to_exists(launches, exists):
+def _audit_launches_to_exists(launches, exists, beginning):
     fails = []
     for (instance, launches) in launches.items():
         if instance in exists:
-            for launch1 in launches:
+            for expected in launches:
                 found = False
-                for launch2 in exists[instance]:
+                for actual in exists[instance]:
-                    if int(launch1['launched_at']) == int(launch2['launched_at']):
+                    if int(expected['launched_at']) == \
+                            int(actual['launched_at']):
                     # HACK (apmelton): Truncate the decimal because we may not
                     #    have the milliseconds.
                         found = True
 
                 if not found:
+                    rec = False
+                    if reconciler:
+                        args = (expected['id'], beginning)
+                        rec = reconciler.missing_exists_for_instance(*args)
                     msg = "Couldn't find exists for launch (%s, %s)"
-                    msg = msg % (instance, launch1['launched_at'])
+                    msg = msg % (instance, expected['launched_at'])
-                    fails.append(['Launch', launch1['id'], msg])
+                    fails.append(['Launch', expected['id'], msg, 'Y' if rec else 'N'])
         else:
+            rec = False
+            if reconciler:
+                args = (launches[0]['id'], beginning)
+                rec = reconciler.missing_exists_for_instance(*args)
             msg = "No exists for instance (%s)" % instance
-            fails.append(['Launch', '-', msg])
+            fails.append(['Launch', '-', msg, 'Y' if rec else 'N'])
     return fails
 
 
@@ -175,8 +193,13 @@ def _launch_audit_for_period(beginning, ending):
         else:
             launches_dict[instance] = [l, ]
 
-    old_launches = models.InstanceUsage.objects.raw(OLD_LAUNCHES_QUERY,
+    # NOTE (apmelton)
-                                                    [beginning, beginning])
+    # Django's safe substitution doesn't allow dict substitution...
+    # Thus, we send it 'beginning' three times...
+    old_launches = models.InstanceUsage.objects\
+                         .raw(OLD_LAUNCHES_QUERY,
+                              [beginning, beginning, beginning])
+
     old_launches_dict = {}
     for launch in old_launches:
         instance = launch.instance
@@ -205,7 +228,8 @@ def _launch_audit_for_period(beginning, ending):
             exists_dict[instance] = [e, ]
 
     launch_to_exists_fails = _audit_launches_to_exists(launches_dict,
-                                                       exists_dict)
+                                                       exists_dict,
+                                                       beginning)
 
     return launch_to_exists_fails, new_launches.count(), len(old_launches_dict)
 
@@ -222,11 +246,11 @@ def audit_for_period(beginning, ending):
 
     summary = {
         'verifier': verify_summary,
-        'launch_fails': {
+        'launch_summary': {
-            'total_failures': len(detail),
             'new_launches': new_count,
-            'old_launches': old_count
+            'old_launches': old_count,
-        }
+            'failures': len(detail)
+        },
     }
 
     details = {
@@ -276,7 +300,7 @@ def store_results(start, end, summary, details):
 
 def make_json_report(summary, details):
     report = [{'summary': summary},
-              ['Object', 'ID', 'Error Description']]
+              ['Object', 'ID', 'Error Description', 'Reconciled?']]
     report.extend(details['exist_fails'])
     report.extend(details['launch_fails'])
     return json.dumps(report)
@@ -302,8 +326,20 @@ if __name__ == '__main__':
                         help="If set to true, report will be stored. "
                              "Otherwise, it will just be printed",
                         type=bool, default=False)
+    parser.add_argument('--reconcile',
+                        help="Enabled reconciliation",
+                        type=bool, default=False)
+    parser.add_argument('--reconciler_config',
+                        help="Location of the reconciler config file",
+                        type=str,
+                        default='/etc/stacktach/reconciler-config.json')
     args = parser.parse_args()
 
+    if args.reconcile:
+        with open(args.reconciler_config) as f:
+            reconciler_config = json.load(f)
+            reconciler = Reconciler(reconciler_config)
+
     if args.utcdatetime is not None:
         time = args.utcdatetime
     else:

stacktach/migrations/0004_create_instancereconcile.py

@@ -0,0 +1,160 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'InstanceReconcile'
+        db.create_table(u'stacktach_instancereconcile', (
+            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('row_created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
+            ('row_updated', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)),
+            ('instance', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)),
+            ('launched_at', self.gf('django.db.models.fields.DecimalField')(null=True, max_digits=20, decimal_places=6, db_index=True)),
+            ('deleted_at', self.gf('django.db.models.fields.DecimalField')(null=True, max_digits=20, decimal_places=6, db_index=True)),
+            ('instance_type_id', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)),
+            ('source', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=150, null=True, blank=True)),
+        ))
+        db.send_create_signal(u'stacktach', ['InstanceReconcile'])
+
+
+    def backwards(self, orm):
+        # Deleting model 'InstanceReconcile'
+        db.delete_table(u'stacktach_instancereconcile')
+
+
+    models = {
+        u'stacktach.deployment': {
+            'Meta': {'object_name': 'Deployment'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        u'stacktach.instancedeletes': {
+            'Meta': {'object_name': 'InstanceDeletes'},
+            'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
+            'raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.RawData']", 'null': 'True'})
+        },
+        u'stacktach.instanceexists': {
+            'Meta': {'object_name': 'InstanceExists'},
+            'audit_period_beginning': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
+            'audit_period_ending': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
+            'delete': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.InstanceDeletes']"}),
+            'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
+            'fail_reason': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '300', 'null': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'instance_type_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
+            'message_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'os_architecture': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'os_distro': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'os_version': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'raw': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.RawData']"}),
+            'rax_options': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'send_status': ('django.db.models.fields.IntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True'}),
+            'status': ('django.db.models.fields.CharField', [], {'default': "'pending'", 'max_length': '50', 'db_index': 'True'}),
+            'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'usage': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.InstanceUsage']"})
+        },
+        u'stacktach.instancereconcile': {
+            'Meta': {'object_name': 'InstanceReconcile'},
+            'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'instance_type_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
+            'row_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'row_updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'source': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '150', 'null': 'True', 'blank': 'True'})
+        },
+        u'stacktach.instanceusage': {
+            'Meta': {'object_name': 'InstanceUsage'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'instance_type_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
+            'os_architecture': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'os_distro': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'os_version': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'rax_options': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'request_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'})
+        },
+        u'stacktach.jsonreport': {
+            'Meta': {'object_name': 'JsonReport'},
+            'created': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'json': ('django.db.models.fields.TextField', [], {}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
+            'period_end': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+            'period_start': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+            'version': ('django.db.models.fields.IntegerField', [], {'default': '1'})
+        },
+        u'stacktach.lifecycle': {
+            'Meta': {'object_name': 'Lifecycle'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'last_raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.RawData']", 'null': 'True'}),
+            'last_state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'last_task_state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'})
+        },
+        u'stacktach.rawdata': {
+            'Meta': {'object_name': 'RawData'},
+            'deployment': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Deployment']"}),
+            'event': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'host': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'image_type': ('django.db.models.fields.IntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True'}),
+            'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'json': ('django.db.models.fields.TextField', [], {}),
+            'old_state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '20', 'null': 'True', 'blank': 'True'}),
+            'old_task': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '30', 'null': 'True', 'blank': 'True'}),
+            'publisher': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}),
+            'request_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'routing_key': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'service': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '20', 'null': 'True', 'blank': 'True'}),
+            'task': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '30', 'null': 'True', 'blank': 'True'}),
+            'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'when': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'})
+        },
+        u'stacktach.rawdataimagemeta': {
+            'Meta': {'object_name': 'RawDataImageMeta'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'os_architecture': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'os_distro': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'os_version': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.RawData']"}),
+            'rax_options': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'})
+        },
+        u'stacktach.requesttracker': {
+            'Meta': {'object_name': 'RequestTracker'},
+            'completed': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+            'duration': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_timing': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Timing']", 'null': 'True'}),
+            'lifecycle': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Lifecycle']"}),
+            'request_id': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
+            'start': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'})
+        },
+        u'stacktach.timing': {
+            'Meta': {'object_name': 'Timing'},
+            'diff': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
+            'end_raw': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.RawData']"}),
+            'end_when': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'lifecycle': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Lifecycle']"}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
+            'start_raw': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.RawData']"}),
+            'start_when': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6'})
+        }
+    }
+
+    complete_apps = ['stacktach']

stacktach/models.py

@@ -101,6 +101,13 @@ class InstanceUsage(models.Model):
     os_version = models.TextField(null=True, blank=True)
     rax_options = models.TextField(null=True, blank=True)
 
+    def deployment(self):
+        raws = RawData.objects.filter(request_id=self.request_id)
+        if raws.count() == 0:
+            return False
+        raw = raws[0]
+        return raw.deployment
+
 class InstanceDeletes(models.Model):
     instance = models.CharField(max_length=50, null=True,
                                 blank=True, db_index=True)
@@ -111,15 +118,34 @@ class InstanceDeletes(models.Model):
     raw = models.ForeignKey(RawData, null=True)
 
 
+class InstanceReconcile(models.Model):
+    row_created = models.DateTimeField(auto_now_add=True)
+    row_updated = models.DateTimeField(auto_now=True)
+    instance = models.CharField(max_length=50, null=True,
+                                blank=True, db_index=True)
+    launched_at = models.DecimalField(null=True, max_digits=20,
+                                      decimal_places=6, db_index=True)
+    deleted_at = models.DecimalField(null=True, max_digits=20,
+                                     decimal_places=6, db_index=True)
+    instance_type_id = models.CharField(max_length=50,
+                                        null=True,
+                                        blank=True,
+                                        db_index=True)
+    source = models.CharField(max_length=150, null=True,
+                              blank=True, db_index=True)
+
+
 class InstanceExists(models.Model):
     PENDING = 'pending'
     VERIFYING = 'verifying'
     VERIFIED = 'verified'
+    RECONCILED = 'reconciled'
     FAILED = 'failed'
     STATUS_CHOICES = [
         (PENDING, 'Pending Verification'),
         (VERIFYING, 'Currently Being Verified'),
         (VERIFIED, 'Passed Verification'),
+        (RECONCILED, 'Passed Verification After Reconciliation'),
         (FAILED, 'Failed Verification'),
     ]
     instance = models.CharField(max_length=50, null=True,

stacktach/reconciler/__init__.py

@@ -0,0 +1,107 @@
+# Copyright (c) 2013 - Rackspace Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+import json
+
+from stacktach import models
+from stacktach.reconciler import exceptions
+from stacktach.reconciler import nova
+
+DEFAULT_CLIENT = nova.JSONBridgeClient
+
+CONFIG = {
+    'client_class': 'JSONBridgeClient',
+    'client': {
+        'url': 'http://stack.dev.ramielrowe.com:8080/query/',
+        'username': '',
+        'password': '',
+        'databases': {
+            'RegionOne': 'nova',
+        }
+    },
+    'region_mapping_loc': '/etc/stacktach/region_mapping.json'
+}
+
+
+class Reconciler(object):
+
+    def __init__(self, config, client=None, region_mapping=None):
+        self.config = config
+        self.client = (client or Reconciler.load_client(config))
+        self.region_mapping = (region_mapping or
+                               Reconciler.load_region_mapping(config))
+
+    @classmethod
+    def load_client(cls, config):
+        client_class = config.get('client_class')
+        if client_class == 'JSONBridgeClient':
+            return nova.JSONBridgeClient(config['client'])
+        else:
+            return DEFAULT_CLIENT(config['client'])
+
+    @classmethod
+    def load_region_mapping(cls, config):
+        with open(config['region_mapping_loc']) as f:
+            return json.load(f)
+
+    def _region_for_launch(self, launch):
+        deployment = launch.deployment()
+        if deployment:
+            deployment_name = str(deployment.name)
+            if deployment_name in self.region_mapping:
+                return self.region_mapping[deployment_name]
+            else:
+                return False
+        else:
+            return False
+
+    def _reconcile_instance(self, launch, src,
+                            launched_at=None, deleted_at=None,
+                            instance_type_id=None):
+        values = {
+            'instance': launch.instance,
+            'launched_at': (launched_at or launch.launched_at),
+            'deleted_at': deleted_at,
+            'instance_type_id': (instance_type_id or launch.instance_type_id),
+            'source': 'reconciler:%s' % src,
+        }
+        models.InstanceReconcile(**values).save()
+
+    def missing_exists_for_instance(self, launched_id,
+                                    period_beginning):
+        reconciled = False
+        launch = models.InstanceUsage.objects.get(id=launched_id)
+        region = self._region_for_launch(launch)
+        try:
+            instance = self.client.get_instance(region, launch.instance)
+            if instance['deleted'] and instance['deleted_at'] is not None:
+                # Check to see if instance has been deleted
+                deleted_at = instance['deleted_at']
+
+                if deleted_at < period_beginning:
+                    # Check to see if instance was deleted before period.
+                    # If so, we shouldn't expect an exists.
+                    self._reconcile_instance(launch, self.client.src_str,
+                                             deleted_at=instance['deleted_at'])
+                    reconciled = True
+        except exceptions.NotFound:
+            reconciled = False
+
+        return reconciled

stacktach/reconciler/exceptions.py

@@ -0,0 +1,3 @@
+class NotFound(Exception):
+    def __init__(self, message="NotFound"):
+        self.message = message

stacktach/reconciler/nova.py

@@ -0,0 +1,54 @@
+import json
+
+import requests
+
+from stacktach import utils as stackutils
+from stacktach.reconciler import exceptions
+from stacktach.reconciler.utils import empty_reconciler_instance
+
+
+GET_INSTANCE_QUERY = "SELECT * FROM instances where uuid ='%s';"
+
+
+class JSONBridgeClient(object):
+    src_str = 'json_bridge:nova_db'
+
+    def __init__(self, config):
+        self.config = config
+
+    def _url_for_region(self, region):
+        return self.config['url'] + self.config['databases'][region]
+
+    def _do_query(self, region, query):
+        data = {'sql': query}
+        credentials = (self.config['username'], self.config['password'])
+        return requests.post(self._url_for_region(region), data,
+                             verify=False, auth=credentials).json()
+
+    def _to_reconciler_instance(self, instance):
+        r_instance = empty_reconciler_instance()
+        r_instance.update({
+            'id': instance['uuid'],
+            'instance_type_id': instance['instance_type_id'],
+        })
+
+        if instance['launched_at'] is not None:
+            launched_at = stackutils.str_time_to_unix(instance['launched_at'])
+            r_instance['launched_at'] = launched_at
+
+        if instance['terminated_at'] is not None:
+            deleted_at = stackutils.str_time_to_unix(instance['terminated_at'])
+            r_instance['deleted_at'] = deleted_at
+
+        if instance['deleted'] != 0:
+            r_instance['deleted'] = True
+
+        return r_instance
+
+    def get_instance(self, region, uuid):
+        results = self._do_query(region, GET_INSTANCE_QUERY % uuid)['result']
+        if len(results) > 0:
+            return self._to_reconciler_instance(results[0])
+        else:
+            msg = "Couldn't find instance (%s) using JSON Bridge in region (%s)"
+            raise exceptions.NotFound(msg % (uuid, region))

stacktach/reconciler/utils.py

@@ -0,0 +1,9 @@
+def empty_reconciler_instance():
+    r_instance = {
+        'id': None,
+        'launched_at': None,
+        'deleted': False,
+        'deleted_at': None,
+        'instance_type_ud': None
+    }
+    return r_instance

tests/unit/test_reconciler.py

@@ -0,0 +1,258 @@
+# Copyright (c) 2013 - Rackspace Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+import datetime
+import unittest
+
+import mox
+import requests
+
+from stacktach import models
+from stacktach import reconciler
+from stacktach import utils as stackutils
+from stacktach.reconciler import exceptions
+from stacktach.reconciler import nova
+from tests.unit import utils
+from tests.unit.utils import INSTANCE_ID_1
+
+region_mapping = {
+    'RegionOne.prod.cell1': 'RegionOne',
+    'RegionTwo.prod.cell1': 'RegionTwo',
+}
+
+
+class ReconcilerTestCase(unittest.TestCase):
+    def setUp(self):
+        self.mox = mox.Mox()
+        self.client = self.mox.CreateMockAnything()
+        self.client.src_str = 'mocked_client'
+        self.reconciler = reconciler.Reconciler({},
+                                                client=self.client,
+                                                region_mapping=region_mapping)
+        self.mox.StubOutWithMock(models, 'RawData', use_mock_anything=True)
+        models.RawData.objects = self.mox.CreateMockAnything()
+        self.mox.StubOutWithMock(models, 'Deployment', use_mock_anything=True)
+        models.Deployment.objects = self.mox.CreateMockAnything()
+        self.mox.StubOutWithMock(models, 'Lifecycle', use_mock_anything=True)
+        models.Lifecycle.objects = self.mox.CreateMockAnything()
+        self.mox.StubOutWithMock(models, 'Timing', use_mock_anything=True)
+        models.Timing.objects = self.mox.CreateMockAnything()
+        self.mox.StubOutWithMock(models, 'RequestTracker',
+                                 use_mock_anything=True)
+        models.RequestTracker.objects = self.mox.CreateMockAnything()
+        self.mox.StubOutWithMock(models, 'InstanceUsage',
+                                 use_mock_anything=True)
+        models.InstanceUsage.objects = self.mox.CreateMockAnything()
+        self.mox.StubOutWithMock(models, 'InstanceReconcile',
+                                 use_mock_anything=True)
+        models.InstanceReconcile.objects = self.mox.CreateMockAnything()
+        self.mox.StubOutWithMock(models, 'InstanceDeletes',
+                                 use_mock_anything=True)
+        models.InstanceDeletes.objects = self.mox.CreateMockAnything()
+        self.mox.StubOutWithMock(models, 'InstanceExists',
+                                 use_mock_anything=True)
+        models.InstanceExists.objects = self.mox.CreateMockAnything()
+        self.mox.StubOutWithMock(models, 'JsonReport', use_mock_anything=True)
+        models.JsonReport.objects = self.mox.CreateMockAnything()
+
+    def tearDown(self):
+        self.mox.UnsetStubs()
+
+    def _fake_reconciler_instance(self, uuid=INSTANCE_ID_1, launched_at=None,
+                                  deleted_at=None, deleted=False,
+                                  instance_type_id=1):
+        return {
+            'id': uuid,
+            'launched_at': launched_at,
+            'deleted_at': deleted_at,
+            'deleted': deleted,
+            'instance_type_id': instance_type_id
+        }
+
+    def test_load_client_json_bridge(self):
+        mock_config = self.mox.CreateMockAnything()
+        config = {'client_class': 'JSONBridgeClient', 'client': mock_config}
+        nova.JSONBridgeClient(mock_config)
+        self.mox.ReplayAll()
+        reconciler.Reconciler.load_client(config)
+        self.mox.VerifyAll()
+
+    def test_load_client_no_class_loads_default_class(self):
+        mock_config = self.mox.CreateMockAnything()
+        config = {'client': mock_config}
+        nova.JSONBridgeClient(mock_config)
+        self.mox.ReplayAll()
+        reconciler.Reconciler.load_client(config)
+        self.mox.VerifyAll()
+
+    def test_load_client_incorrect_class_loads_default_class(self):
+        mock_config = self.mox.CreateMockAnything()
+        config = {'client_class': 'BadConfigValue', 'client': mock_config}
+        nova.JSONBridgeClient(mock_config)
+        self.mox.ReplayAll()
+        reconciler.Reconciler.load_client(config)
+        self.mox.VerifyAll()
+
+    def test_region_for_launch(self):
+        launch = self.mox.CreateMockAnything()
+        deployment = self.mox.CreateMockAnything()
+        deployment.name = 'RegionOne.prod.cell1'
+        launch.deployment().AndReturn(deployment)
+        self.mox.ReplayAll()
+        region = self.reconciler._region_for_launch(launch)
+        self.assertEqual('RegionOne', region)
+        self.mox.VerifyAll()
+
+    def test_region_for_launch_no_mapping(self):
+        launch = self.mox.CreateMockAnything()
+        deployment = self.mox.CreateMockAnything()
+        deployment.name = 'RegionOne.prod.cell2'
+        launch.deployment().AndReturn(deployment)
+        self.mox.ReplayAll()
+        region = self.reconciler._region_for_launch(launch)
+        self.assertFalse(region)
+        self.mox.VerifyAll()
+
+    def test_region_for_launch_no_raws(self):
+        launch = self.mox.CreateMockAnything()
+        launch.deployment()
+        self.mox.ReplayAll()
+        region = self.reconciler._region_for_launch(launch)
+        self.assertFalse(region)
+        self.mox.VerifyAll()
+
+    def test_missing_exists_for_instance(self):
+        launch_id = 1
+        beginning_d = utils.decimal_utc()
+        launch = self.mox.CreateMockAnything()
+        launch.instance = INSTANCE_ID_1
+        launch.launched_at = beginning_d - (60*60)
+        launch.instance_type_id = 1
+        models.InstanceUsage.objects.get(id=launch_id).AndReturn(launch)
+        deployment = self.mox.CreateMockAnything()
+        launch.deployment().AndReturn(deployment)
+        deployment.name = 'RegionOne.prod.cell1'
+        deleted_at = beginning_d - (60*30)
+        rec_inst = self._fake_reconciler_instance(deleted=True,
+                                                  deleted_at=deleted_at)
+        self.client.get_instance('RegionOne', INSTANCE_ID_1).AndReturn(rec_inst)
+        reconcile_vals = {
+            'instance': launch.instance,
+            'launched_at': launch.launched_at,
+            'deleted_at': deleted_at,
+            'instance_type_id': launch.instance_type_id,
+            'source': 'reconciler:mocked_client'
+        }
+        result = self.mox.CreateMockAnything()
+        models.InstanceReconcile(**reconcile_vals).AndReturn(result)
+        result.save()
+        self.mox.ReplayAll()
+        result = self.reconciler.missing_exists_for_instance(launch_id,
+                                                             beginning_d)
+        self.assertTrue(result)
+        self.mox.VerifyAll()
+
+    def test_missing_exists_for_instance_not_found(self):
+        launch_id = 1
+        beginning_d = utils.decimal_utc()
+        launch = self.mox.CreateMockAnything()
+        launch.instance = INSTANCE_ID_1
+        launch.launched_at = beginning_d - (60*60)
+        launch.instance_type_id = 1
+        models.InstanceUsage.objects.get(id=launch_id).AndReturn(launch)
+        deployment = self.mox.CreateMockAnything()
+        launch.deployment().AndReturn(deployment)
+        deployment.name = 'RegionOne.prod.cell1'
+        ex = exceptions.NotFound()
+        self.client.get_instance('RegionOne', INSTANCE_ID_1).AndRaise(ex)
+        self.mox.ReplayAll()
+        result = self.reconciler.missing_exists_for_instance(launch_id,
+                                                             beginning_d)
+        self.assertFalse(result)
+        self.mox.VerifyAll()
+
+
+json_bridge_config = {
+    'url': 'http://json_bridge.example.com/query/',
+    'username': 'user',
+    'password': 'pass',
+    'databases': {
+        'RegionOne': 'nova',
+    }
+}
+
+
+class NovaJSONBridgeClientTestCase(unittest.TestCase):
+    def setUp(self):
+        self.mox = mox.Mox()
+        self.client = nova.JSONBridgeClient(json_bridge_config)
+        self.mox.StubOutWithMock(requests, 'post')
+
+    def tearDown(self):
+        self.mox.UnsetStubs()
+
+    def mock_for_query(self, database, query, results):
+        url = json_bridge_config['url'] + database
+        data = {'sql': query}
+        auth = (json_bridge_config['username'], json_bridge_config['password'])
+        result = {'result': results}
+        response = self.mox.CreateMockAnything()
+        requests.post(url, data, auth=auth, verify=False)\
+                .AndReturn(response)
+        response.json().AndReturn(result)
+
+    def _fake_instance(self, uuid=INSTANCE_ID_1, launched_at=None,
+                      terminated_at=None, deleted=0, instance_type_id=1):
+        return {
+            'uuid': uuid,
+            'launched_at': launched_at,
+            'terminated_at': terminated_at,
+            'deleted': deleted,
+            'instance_type_id': instance_type_id
+        }
+
+    def test_get_instance(self):
+        launched_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=5)
+        launched_at = str(launched_at)
+        terminated_at = str(datetime.datetime.utcnow())
+        results = [self._fake_instance(launched_at=launched_at,
+                                       terminated_at=terminated_at,
+                                       deleted=True)]
+        self.mock_for_query('nova', nova.GET_INSTANCE_QUERY % INSTANCE_ID_1,
+                            results)
+        self.mox.ReplayAll()
+        instance = self.client.get_instance('RegionOne', INSTANCE_ID_1)
+        self.assertIsNotNone(instance)
+        self.assertEqual(instance['id'], INSTANCE_ID_1)
+        self.assertEqual(instance['instance_type_id'], 1)
+        launched_at_dec = stackutils.str_time_to_unix(launched_at)
+        self.assertEqual(instance['launched_at'], launched_at_dec)
+        terminated_at_dec = stackutils.str_time_to_unix(terminated_at)
+        self.assertEqual(instance['deleted_at'], terminated_at_dec)
+        self.assertTrue(instance['deleted'])
+        self.mox.VerifyAll()
+
+    def test_get_instance_not_found(self):
+        self.mock_for_query('nova', nova.GET_INSTANCE_QUERY % INSTANCE_ID_1,
+                            [])
+        self.mox.ReplayAll()
+        self.assertRaises(exceptions.NotFound, self.client.get_instance,
+                          'RegionOne', INSTANCE_ID_1)
+        self.mox.VerifyAll()

tests/unit/test_verifier_db.py

@@ -71,6 +71,9 @@ class VerifierTestCase(unittest.TestCase):
         self.mox.StubOutWithMock(models, 'InstanceDeletes',
                                  use_mock_anything=True)
         models.InstanceDeletes.objects = self.mox.CreateMockAnything()
+        self.mox.StubOutWithMock(models, 'InstanceReconcile',
+                                 use_mock_anything=True)
+        models.InstanceReconcile.objects = self.mox.CreateMockAnything()
         self.mox.StubOutWithMock(models, 'InstanceExists',
                                  use_mock_anything=True)
         models.InstanceExists.objects = self.mox.CreateMockAnything()
@@ -448,7 +451,8 @@ class VerifierTestCase(unittest.TestCase):
         dbverifier._verify_for_delete(exist)
         dbverifier._mark_exist_verified(exist)
         self.mox.ReplayAll()
-        dbverifier._verify(exist)
+        result, exists = dbverifier._verify(exist)
+        self.assertTrue(result)
         self.mox.VerifyAll()
 
     def test_verify_no_launched_at(self):
@@ -460,8 +464,12 @@ class VerifierTestCase(unittest.TestCase):
         self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified')
         dbverifier._mark_exist_failed(exist,
                                       reason="Exists without a launched_at")
+        self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data')
+        dbverifier._verify_with_reconciled_data(exist, mox.IgnoreArg())\
+                  .AndRaise(NotFound('InstanceReconcile', {}))
         self.mox.ReplayAll()
-        dbverifier._verify(exist)
+        result, exists = dbverifier._verify(exist)
+        self.assertFalse(result)
         self.mox.VerifyAll()
 
     def test_verify_launch_fail(self):
@@ -473,9 +481,48 @@ class VerifierTestCase(unittest.TestCase):
         self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified')
         verify_exception = VerificationException('test')
         dbverifier._verify_for_launch(exist).AndRaise(verify_exception)
+        self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data')
+        dbverifier._verify_with_reconciled_data(exist, verify_exception)\
+                  .AndRaise(NotFound('InstanceReconcile', {}))
         dbverifier._mark_exist_failed(exist, reason='test')
         self.mox.ReplayAll()
-        dbverifier._verify(exist)
+        result, exists = dbverifier._verify(exist)
+        self.assertFalse(result)
+        self.mox.VerifyAll()
+
+    def test_verify_fail_reconcile_success(self):
+        exist = self.mox.CreateMockAnything()
+        exist.launched_at = decimal.Decimal('1.1')
+        self.mox.StubOutWithMock(dbverifier, '_verify_for_launch')
+        self.mox.StubOutWithMock(dbverifier, '_verify_for_delete')
+        self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed')
+        self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified')
+        verify_exception = VerificationException('test')
+        dbverifier._verify_for_launch(exist).AndRaise(verify_exception)
+        self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data')
+        dbverifier._verify_with_reconciled_data(exist, verify_exception)
+        dbverifier._mark_exist_verified(exist)
+        self.mox.ReplayAll()
+        result, exists = dbverifier._verify(exist)
+        self.assertTrue(result)
+        self.mox.VerifyAll()
+
+    def test_verify_fail_with_reconciled_data_exception(self):
+        exist = self.mox.CreateMockAnything()
+        exist.launched_at = decimal.Decimal('1.1')
+        self.mox.StubOutWithMock(dbverifier, '_verify_for_launch')
+        self.mox.StubOutWithMock(dbverifier, '_verify_for_delete')
+        self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed')
+        self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified')
+        verify_exception = VerificationException('test')
+        dbverifier._verify_for_launch(exist).AndRaise(verify_exception)
+        self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data')
+        dbverifier._verify_with_reconciled_data(exist, verify_exception)\
+                  .AndRaise(Exception())
+        dbverifier._mark_exist_failed(exist, reason='Exception')
+        self.mox.ReplayAll()
+        result, exists = dbverifier._verify(exist)
+        self.assertFalse(result)
         self.mox.VerifyAll()
 
     def test_verify_delete_fail(self):
@@ -488,9 +535,13 @@ class VerifierTestCase(unittest.TestCase):
         verify_exception = VerificationException('test')
         dbverifier._verify_for_launch(exist)
         dbverifier._verify_for_delete(exist).AndRaise(verify_exception)
+        self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data')
+        dbverifier._verify_with_reconciled_data(exist, verify_exception)\
+                  .AndRaise(NotFound('InstanceReconcile', {}))
         dbverifier._mark_exist_failed(exist, reason='test')
         self.mox.ReplayAll()
-        dbverifier._verify(exist)
+        result, exists = dbverifier._verify(exist)
+        self.assertFalse(result)
         self.mox.VerifyAll()
 
     def test_verify_exception_during_launch(self):
@@ -503,7 +554,8 @@ class VerifierTestCase(unittest.TestCase):
         dbverifier._verify_for_launch(exist).AndRaise(Exception())
         dbverifier._mark_exist_failed(exist, reason='Exception')
         self.mox.ReplayAll()
-        dbverifier._verify(exist)
+        result, exists = dbverifier._verify(exist)
+        self.assertFalse(result)
         self.mox.VerifyAll()
 
     def test_verify_exception_during_delete(self):
@@ -517,7 +569,8 @@ class VerifierTestCase(unittest.TestCase):
         dbverifier._verify_for_delete(exist).AndRaise(Exception())
         dbverifier._mark_exist_failed(exist, reason='Exception')
         self.mox.ReplayAll()
-        dbverifier._verify(exist)
+        result, exists = dbverifier._verify(exist)
+        self.assertFalse(result)
         self.mox.VerifyAll()
 
     def test_verify_for_range_without_callback(self):

verifier/dbverifier.py

@@ -69,6 +69,15 @@ def _find_launch(instance, launched):
     return models.InstanceUsage.objects.filter(**params)
 
 
+def _find_reconcile(instance, launched):
+    start = launched - datetime.timedelta(microseconds=launched.microsecond)
+    end = start + datetime.timedelta(microseconds=999999)
+    params = {'instance': instance,
+              'launched_at__gte': dt.dt_to_decimal(start),
+              'launched_at__lte': dt.dt_to_decimal(end)}
+    return models.InstanceReconcile.objects.filter(**params)
+
+
 def _find_delete(instance, launched, deleted_max=None):
     start = launched - datetime.timedelta(microseconds=launched.microsecond)
     end = start + datetime.timedelta(microseconds=999999)
@@ -80,8 +89,16 @@ def _find_delete(instance, launched, deleted_max=None):
     return models.InstanceDeletes.objects.filter(**params)
 
 
-def _mark_exist_verified(exist):
+def _mark_exist_verified(exist,
-    exist.status = models.InstanceExists.VERIFIED
+                         reconciled=False,
+                         reason=None):
+    if not reconciled:
+        exist.status = models.InstanceExists.VERIFIED
+    else:
+        exist.status = models.InstanceExists.RECONCILED
+        if reason is not None:
+            exist.fail_reason = reason
+
     exist.save()
 
 
@@ -152,10 +169,11 @@ def _verify_field_mismatch(exists, launch):
                             launch.os_distro)
 
 
-def _verify_for_launch(exist):
+def _verify_for_launch(exist, launch=None, launch_type="InstanceUsage"):
-    if exist.usage:
+
+    if not launch and exist.usage:
         launch = exist.usage
-    else:
+    elif not launch:
         if models.InstanceUsage.objects\
                  .filter(instance=exist.instance).count() > 0:
             launches = _find_launch(exist.instance,
@@ -166,23 +184,22 @@ def _verify_for_launch(exist):
                 'launched_at': exist.launched_at
             }
             if count > 1:
-                raise AmbiguousResults('InstanceUsage', query)
+                raise AmbiguousResults(launch_type, query)
             elif count == 0:
-                raise NotFound('InstanceUsage', query)
+                raise NotFound(launch_type, query)
             launch = launches[0]
         else:
-            raise NotFound('InstanceUsage', {'instance': exist.instance})
+            raise NotFound(launch_type, {'instance': exist.instance})
 
     _verify_field_mismatch(exist, launch)
 
 
-def _verify_for_delete(exist):
+def _verify_for_delete(exist, delete=None, delete_type="InstanceDelete"):
 
-    delete = None
+    if not delete and exist.delete:
-    if exist.delete:
         # We know we have a delete and we have it's id
         delete = exist.delete
-    else:
+    elif not delete:
         if exist.deleted_at:
             # We received this exists before the delete, go find it
             deletes = _find_delete(exist.instance,
@@ -194,7 +211,7 @@ def _verify_for_delete(exist):
                     'instance': exist.instance,
                     'launched_at': exist.launched_at
                 }
-                raise NotFound('InstanceDelete', query)
+                raise NotFound(delete_type, query)
         else:
             # We don't know if this is supposed to have a delete or not.
             # Thus, we need to check if we have a delete for this instance.
@@ -206,7 +223,7 @@ def _verify_for_delete(exist):
             deleted_at_max = dt.dt_from_decimal(exist.audit_period_ending)
             deletes = _find_delete(exist.instance, launched_at, deleted_at_max)
             if deletes.count() > 0:
-                reason = 'Found InstanceDeletes for non-delete exist'
+                reason = 'Found %ss for non-delete exist' % delete_type
                 raise VerificationException(reason)
 
     if delete:
@@ -221,6 +238,31 @@ def _verify_for_delete(exist):
                                 delete.deleted_at)
 
 
+def _verify_with_reconciled_data(exist, ex):
+    if not exist.launched_at:
+        raise VerificationException("Exists without a launched_at")
+
+    query = models.InstanceReconcile.objects.filter(instance=exist.instance)
+    if query.count() > 0:
+        recs = _find_reconcile(exist.instance,
+                               dt.dt_from_decimal(exist.launched_at))
+        search_query = {'instance': exist.instance,
+                        'launched_at': exist.launched_at}
+        count = recs.count()
+        if count > 1:
+            raise AmbiguousResults('InstanceReconcile', search_query)
+        elif count == 0:
+            raise NotFound('InstanceReconcile', search_query)
+        reconcile = recs[0]
+    else:
+        raise NotFound('InstanceReconcile', {'instance': exist.instance})
+
+    _verify_for_launch(exist, launch=reconcile,
+                       launch_type="InstanceReconcile")
+    _verify_for_delete(exist, delete=reconcile,
+                       delete_type="InstanceReconcile")
+
+
 def _verify(exist):
     verified = False
     try:
@@ -232,8 +274,23 @@ def _verify(exist):
 
         verified = True
         _mark_exist_verified(exist)
-    except VerificationException, e:
+    except VerificationException, orig_e:
-        _mark_exist_failed(exist, reason=str(e))
+        # Something is wrong with the InstanceUsage record
+        try:
+            # Attempt to verify against reconciled data
+            _verify_with_reconciled_data(exist, orig_e)
+            verified = True
+            _mark_exist_verified(exist)
+        except NotFound, rec_e:
+            # No reconciled data, just mark it failed
+            _mark_exist_failed(exist, reason=str(orig_e))
+        except VerificationException, rec_e:
+            # Verification failed against reconciled data, mark it failed
+            #    using the second failure.
+            _mark_exist_failed(exist, reason=str(rec_e))
+        except Exception, rec_e:
+            _mark_exist_failed(exist, reason=rec_e.__class__.__name__)
+            LOG.exception(rec_e)
     except Exception, e:
         _mark_exist_failed(exist, reason=e.__class__.__name__)
         LOG.exception(e)