diff --git a/etc/sample_reconciler_config.json b/etc/sample_reconciler_config.json new file mode 100644 index 0000000..0a19be9 --- /dev/null +++ b/etc/sample_reconciler_config.json @@ -0,0 +1,13 @@ +{ + "client_class": "JSONBridgeClient", + "client": { + "url": "http://jsonbridge.example.com:8080/query/", + "username": "bridgeuser", + "password": "super_secure_password", + "regions": { + "RegionOne": "nova-regionone", + "RegionTwo": "nova-regiontwo" + } + }, + "region_mapping_loc": "etc/sample_region_mapping.json" +} \ No newline at end of file diff --git a/etc/sample_region_mapping.json b/etc/sample_region_mapping.json new file mode 100644 index 0000000..675c817 --- /dev/null +++ b/etc/sample_region_mapping.json @@ -0,0 +1,6 @@ +{ + "RegionOne.dev.global": "RegionOne", + "RegionOne.dev.cell1": "RegionOne", + "RegionTwo.dev.global": "RegionTwo", + "RegionTwo.dev.cell1": "RegionTwo" +} diff --git a/reports/nova_usage_audit.py b/reports/nova_usage_audit.py index 6a7eff4..a4e54f3 100644 --- a/reports/nova_usage_audit.py +++ b/reports/nova_usage_audit.py @@ -30,12 +30,21 @@ from django.db.models import F from stacktach import datetime_to_decimal as dt from stacktach import models +from stacktach.reconciler import Reconciler -OLD_LAUNCHES_QUERY = "select * from stacktach_instanceusage " \ - "where launched_at is not null and " \ - "launched_at < %s and instance not in " \ - "(select distinct(instance) " \ - "from stacktach_instancedeletes where deleted_at < %s)" +OLD_LAUNCHES_QUERY = """ +select * from stacktach_instanceusage where + launched_at is not null and + launched_at < %s and + instance not in + (select distinct(instance) + from stacktach_instancedeletes where + deleted_at < %s union + select distinct(instance) + from stacktach_instancereconcile where + deleted_at < %s);""" + +reconciler = None def _get_new_launches(beginning, ending): @@ -63,25 +72,34 @@ def _get_exists(beginning, ending): return models.InstanceExists.objects.filter(**filters) -def _audit_launches_to_exists(launches, exists): +def _audit_launches_to_exists(launches, exists, beginning): fails = [] for (instance, launches) in launches.items(): if instance in exists: - for launch1 in launches: + for expected in launches: found = False - for launch2 in exists[instance]: - if int(launch1['launched_at']) == int(launch2['launched_at']): + for actual in exists[instance]: + if int(expected['launched_at']) == \ + int(actual['launched_at']): # HACK (apmelton): Truncate the decimal because we may not # have the milliseconds. found = True if not found: + rec = False + if reconciler: + args = (expected['id'], beginning) + rec = reconciler.missing_exists_for_instance(*args) msg = "Couldn't find exists for launch (%s, %s)" - msg = msg % (instance, launch1['launched_at']) - fails.append(['Launch', launch1['id'], msg]) + msg = msg % (instance, expected['launched_at']) + fails.append(['Launch', expected['id'], msg, 'Y' if rec else 'N']) else: + rec = False + if reconciler: + args = (launches[0]['id'], beginning) + rec = reconciler.missing_exists_for_instance(*args) msg = "No exists for instance (%s)" % instance - fails.append(['Launch', '-', msg]) + fails.append(['Launch', '-', msg, 'Y' if rec else 'N']) return fails @@ -175,8 +193,13 @@ def _launch_audit_for_period(beginning, ending): else: launches_dict[instance] = [l, ] - old_launches = models.InstanceUsage.objects.raw(OLD_LAUNCHES_QUERY, - [beginning, beginning]) + # NOTE (apmelton) + # Django's safe substitution doesn't allow dict substitution... + # Thus, we send it 'beginning' three times... + old_launches = models.InstanceUsage.objects\ + .raw(OLD_LAUNCHES_QUERY, + [beginning, beginning, beginning]) + old_launches_dict = {} for launch in old_launches: instance = launch.instance @@ -205,7 +228,8 @@ def _launch_audit_for_period(beginning, ending): exists_dict[instance] = [e, ] launch_to_exists_fails = _audit_launches_to_exists(launches_dict, - exists_dict) + exists_dict, + beginning) return launch_to_exists_fails, new_launches.count(), len(old_launches_dict) @@ -222,11 +246,11 @@ def audit_for_period(beginning, ending): summary = { 'verifier': verify_summary, - 'launch_fails': { - 'total_failures': len(detail), + 'launch_summary': { 'new_launches': new_count, - 'old_launches': old_count - } + 'old_launches': old_count, + 'failures': len(detail) + }, } details = { @@ -276,7 +300,7 @@ def store_results(start, end, summary, details): def make_json_report(summary, details): report = [{'summary': summary}, - ['Object', 'ID', 'Error Description']] + ['Object', 'ID', 'Error Description', 'Reconciled?']] report.extend(details['exist_fails']) report.extend(details['launch_fails']) return json.dumps(report) @@ -302,8 +326,20 @@ if __name__ == '__main__': help="If set to true, report will be stored. " "Otherwise, it will just be printed", type=bool, default=False) + parser.add_argument('--reconcile', + help="Enabled reconciliation", + type=bool, default=False) + parser.add_argument('--reconciler_config', + help="Location of the reconciler config file", + type=str, + default='/etc/stacktach/reconciler-config.json') args = parser.parse_args() + if args.reconcile: + with open(args.reconciler_config) as f: + reconciler_config = json.load(f) + reconciler = Reconciler(reconciler_config) + if args.utcdatetime is not None: time = args.utcdatetime else: diff --git a/stacktach/migrations/0004_create_instancereconcile.py b/stacktach/migrations/0004_create_instancereconcile.py new file mode 100644 index 0000000..681c998 --- /dev/null +++ b/stacktach/migrations/0004_create_instancereconcile.py @@ -0,0 +1,160 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'InstanceReconcile' + db.create_table(u'stacktach_instancereconcile', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('row_created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)), + ('row_updated', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)), + ('instance', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + ('launched_at', self.gf('django.db.models.fields.DecimalField')(null=True, max_digits=20, decimal_places=6, db_index=True)), + ('deleted_at', self.gf('django.db.models.fields.DecimalField')(null=True, max_digits=20, decimal_places=6, db_index=True)), + ('instance_type_id', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + ('source', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=150, null=True, blank=True)), + )) + db.send_create_signal(u'stacktach', ['InstanceReconcile']) + + + def backwards(self, orm): + # Deleting model 'InstanceReconcile' + db.delete_table(u'stacktach_instancereconcile') + + + models = { + u'stacktach.deployment': { + 'Meta': {'object_name': 'Deployment'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + u'stacktach.instancedeletes': { + 'Meta': {'object_name': 'InstanceDeletes'}, + 'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.RawData']", 'null': 'True'}) + }, + u'stacktach.instanceexists': { + 'Meta': {'object_name': 'InstanceExists'}, + 'audit_period_beginning': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'audit_period_ending': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'delete': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.InstanceDeletes']"}), + 'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'fail_reason': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '300', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'instance_type_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'message_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'os_architecture': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'os_distro': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'os_version': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'raw': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.RawData']"}), + 'rax_options': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'send_status': ('django.db.models.fields.IntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True'}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'pending'", 'max_length': '50', 'db_index': 'True'}), + 'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'usage': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.InstanceUsage']"}) + }, + u'stacktach.instancereconcile': { + 'Meta': {'object_name': 'InstanceReconcile'}, + 'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'instance_type_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'row_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'row_updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'source': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '150', 'null': 'True', 'blank': 'True'}) + }, + u'stacktach.instanceusage': { + 'Meta': {'object_name': 'InstanceUsage'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'instance_type_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'os_architecture': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'os_distro': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'os_version': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'rax_options': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'request_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}) + }, + u'stacktach.jsonreport': { + 'Meta': {'object_name': 'JsonReport'}, + 'created': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'json': ('django.db.models.fields.TextField', [], {}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}), + 'period_end': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'period_start': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'version': ('django.db.models.fields.IntegerField', [], {'default': '1'}) + }, + u'stacktach.lifecycle': { + 'Meta': {'object_name': 'Lifecycle'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'last_raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.RawData']", 'null': 'True'}), + 'last_state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'last_task_state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}) + }, + u'stacktach.rawdata': { + 'Meta': {'object_name': 'RawData'}, + 'deployment': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Deployment']"}), + 'event': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'host': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'image_type': ('django.db.models.fields.IntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True'}), + 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'json': ('django.db.models.fields.TextField', [], {}), + 'old_state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '20', 'null': 'True', 'blank': 'True'}), + 'old_task': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '30', 'null': 'True', 'blank': 'True'}), + 'publisher': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'request_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'routing_key': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'service': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '20', 'null': 'True', 'blank': 'True'}), + 'task': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '30', 'null': 'True', 'blank': 'True'}), + 'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'when': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}) + }, + u'stacktach.rawdataimagemeta': { + 'Meta': {'object_name': 'RawDataImageMeta'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'os_architecture': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'os_distro': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'os_version': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.RawData']"}), + 'rax_options': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}) + }, + u'stacktach.requesttracker': { + 'Meta': {'object_name': 'RequestTracker'}, + 'completed': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'duration': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_timing': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Timing']", 'null': 'True'}), + 'lifecycle': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Lifecycle']"}), + 'request_id': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}), + 'start': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}) + }, + u'stacktach.timing': { + 'Meta': {'object_name': 'Timing'}, + 'diff': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'end_raw': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.RawData']"}), + 'end_when': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'lifecycle': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Lifecycle']"}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}), + 'start_raw': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.RawData']"}), + 'start_when': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6'}) + } + } + + complete_apps = ['stacktach'] \ No newline at end of file diff --git a/stacktach/models.py b/stacktach/models.py index e03e190..44c5b75 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -101,6 +101,13 @@ class InstanceUsage(models.Model): os_version = models.TextField(null=True, blank=True) rax_options = models.TextField(null=True, blank=True) + def deployment(self): + raws = RawData.objects.filter(request_id=self.request_id) + if raws.count() == 0: + return False + raw = raws[0] + return raw.deployment + class InstanceDeletes(models.Model): instance = models.CharField(max_length=50, null=True, blank=True, db_index=True) @@ -111,15 +118,34 @@ class InstanceDeletes(models.Model): raw = models.ForeignKey(RawData, null=True) +class InstanceReconcile(models.Model): + row_created = models.DateTimeField(auto_now_add=True) + row_updated = models.DateTimeField(auto_now=True) + instance = models.CharField(max_length=50, null=True, + blank=True, db_index=True) + launched_at = models.DecimalField(null=True, max_digits=20, + decimal_places=6, db_index=True) + deleted_at = models.DecimalField(null=True, max_digits=20, + decimal_places=6, db_index=True) + instance_type_id = models.CharField(max_length=50, + null=True, + blank=True, + db_index=True) + source = models.CharField(max_length=150, null=True, + blank=True, db_index=True) + + class InstanceExists(models.Model): PENDING = 'pending' VERIFYING = 'verifying' VERIFIED = 'verified' + RECONCILED = 'reconciled' FAILED = 'failed' STATUS_CHOICES = [ (PENDING, 'Pending Verification'), (VERIFYING, 'Currently Being Verified'), (VERIFIED, 'Passed Verification'), + (RECONCILED, 'Passed Verification After Reconciliation'), (FAILED, 'Failed Verification'), ] instance = models.CharField(max_length=50, null=True, diff --git a/stacktach/reconciler/__init__.py b/stacktach/reconciler/__init__.py new file mode 100644 index 0000000..68b7138 --- /dev/null +++ b/stacktach/reconciler/__init__.py @@ -0,0 +1,107 @@ +# Copyright (c) 2013 - Rackspace Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import json + +from stacktach import models +from stacktach.reconciler import exceptions +from stacktach.reconciler import nova + +DEFAULT_CLIENT = nova.JSONBridgeClient + +CONFIG = { + 'client_class': 'JSONBridgeClient', + 'client': { + 'url': 'http://stack.dev.ramielrowe.com:8080/query/', + 'username': '', + 'password': '', + 'databases': { + 'RegionOne': 'nova', + } + }, + 'region_mapping_loc': '/etc/stacktach/region_mapping.json' +} + + +class Reconciler(object): + + def __init__(self, config, client=None, region_mapping=None): + self.config = config + self.client = (client or Reconciler.load_client(config)) + self.region_mapping = (region_mapping or + Reconciler.load_region_mapping(config)) + + @classmethod + def load_client(cls, config): + client_class = config.get('client_class') + if client_class == 'JSONBridgeClient': + return nova.JSONBridgeClient(config['client']) + else: + return DEFAULT_CLIENT(config['client']) + + @classmethod + def load_region_mapping(cls, config): + with open(config['region_mapping_loc']) as f: + return json.load(f) + + def _region_for_launch(self, launch): + deployment = launch.deployment() + if deployment: + deployment_name = str(deployment.name) + if deployment_name in self.region_mapping: + return self.region_mapping[deployment_name] + else: + return False + else: + return False + + def _reconcile_instance(self, launch, src, + launched_at=None, deleted_at=None, + instance_type_id=None): + values = { + 'instance': launch.instance, + 'launched_at': (launched_at or launch.launched_at), + 'deleted_at': deleted_at, + 'instance_type_id': (instance_type_id or launch.instance_type_id), + 'source': 'reconciler:%s' % src, + } + models.InstanceReconcile(**values).save() + + def missing_exists_for_instance(self, launched_id, + period_beginning): + reconciled = False + launch = models.InstanceUsage.objects.get(id=launched_id) + region = self._region_for_launch(launch) + try: + instance = self.client.get_instance(region, launch.instance) + if instance['deleted'] and instance['deleted_at'] is not None: + # Check to see if instance has been deleted + deleted_at = instance['deleted_at'] + + if deleted_at < period_beginning: + # Check to see if instance was deleted before period. + # If so, we shouldn't expect an exists. + self._reconcile_instance(launch, self.client.src_str, + deleted_at=instance['deleted_at']) + reconciled = True + except exceptions.NotFound: + reconciled = False + + return reconciled diff --git a/stacktach/reconciler/exceptions.py b/stacktach/reconciler/exceptions.py new file mode 100644 index 0000000..e48f49a --- /dev/null +++ b/stacktach/reconciler/exceptions.py @@ -0,0 +1,3 @@ +class NotFound(Exception): + def __init__(self, message="NotFound"): + self.message = message diff --git a/stacktach/reconciler/nova.py b/stacktach/reconciler/nova.py new file mode 100644 index 0000000..59e5955 --- /dev/null +++ b/stacktach/reconciler/nova.py @@ -0,0 +1,54 @@ +import json + +import requests + +from stacktach import utils as stackutils +from stacktach.reconciler import exceptions +from stacktach.reconciler.utils import empty_reconciler_instance + + +GET_INSTANCE_QUERY = "SELECT * FROM instances where uuid ='%s';" + + +class JSONBridgeClient(object): + src_str = 'json_bridge:nova_db' + + def __init__(self, config): + self.config = config + + def _url_for_region(self, region): + return self.config['url'] + self.config['databases'][region] + + def _do_query(self, region, query): + data = {'sql': query} + credentials = (self.config['username'], self.config['password']) + return requests.post(self._url_for_region(region), data, + verify=False, auth=credentials).json() + + def _to_reconciler_instance(self, instance): + r_instance = empty_reconciler_instance() + r_instance.update({ + 'id': instance['uuid'], + 'instance_type_id': instance['instance_type_id'], + }) + + if instance['launched_at'] is not None: + launched_at = stackutils.str_time_to_unix(instance['launched_at']) + r_instance['launched_at'] = launched_at + + if instance['terminated_at'] is not None: + deleted_at = stackutils.str_time_to_unix(instance['terminated_at']) + r_instance['deleted_at'] = deleted_at + + if instance['deleted'] != 0: + r_instance['deleted'] = True + + return r_instance + + def get_instance(self, region, uuid): + results = self._do_query(region, GET_INSTANCE_QUERY % uuid)['result'] + if len(results) > 0: + return self._to_reconciler_instance(results[0]) + else: + msg = "Couldn't find instance (%s) using JSON Bridge in region (%s)" + raise exceptions.NotFound(msg % (uuid, region)) \ No newline at end of file diff --git a/stacktach/reconciler/utils.py b/stacktach/reconciler/utils.py new file mode 100644 index 0000000..b835d25 --- /dev/null +++ b/stacktach/reconciler/utils.py @@ -0,0 +1,9 @@ +def empty_reconciler_instance(): + r_instance = { + 'id': None, + 'launched_at': None, + 'deleted': False, + 'deleted_at': None, + 'instance_type_ud': None + } + return r_instance diff --git a/tests/unit/test_reconciler.py b/tests/unit/test_reconciler.py new file mode 100644 index 0000000..8ec98fa --- /dev/null +++ b/tests/unit/test_reconciler.py @@ -0,0 +1,258 @@ +# Copyright (c) 2013 - Rackspace Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import datetime +import unittest + +import mox +import requests + +from stacktach import models +from stacktach import reconciler +from stacktach import utils as stackutils +from stacktach.reconciler import exceptions +from stacktach.reconciler import nova +from tests.unit import utils +from tests.unit.utils import INSTANCE_ID_1 + +region_mapping = { + 'RegionOne.prod.cell1': 'RegionOne', + 'RegionTwo.prod.cell1': 'RegionTwo', +} + + +class ReconcilerTestCase(unittest.TestCase): + def setUp(self): + self.mox = mox.Mox() + self.client = self.mox.CreateMockAnything() + self.client.src_str = 'mocked_client' + self.reconciler = reconciler.Reconciler({}, + client=self.client, + region_mapping=region_mapping) + self.mox.StubOutWithMock(models, 'RawData', use_mock_anything=True) + models.RawData.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'Deployment', use_mock_anything=True) + models.Deployment.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'Lifecycle', use_mock_anything=True) + models.Lifecycle.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'Timing', use_mock_anything=True) + models.Timing.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'RequestTracker', + use_mock_anything=True) + models.RequestTracker.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'InstanceUsage', + use_mock_anything=True) + models.InstanceUsage.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'InstanceReconcile', + use_mock_anything=True) + models.InstanceReconcile.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'InstanceDeletes', + use_mock_anything=True) + models.InstanceDeletes.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'InstanceExists', + use_mock_anything=True) + models.InstanceExists.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'JsonReport', use_mock_anything=True) + models.JsonReport.objects = self.mox.CreateMockAnything() + + def tearDown(self): + self.mox.UnsetStubs() + + def _fake_reconciler_instance(self, uuid=INSTANCE_ID_1, launched_at=None, + deleted_at=None, deleted=False, + instance_type_id=1): + return { + 'id': uuid, + 'launched_at': launched_at, + 'deleted_at': deleted_at, + 'deleted': deleted, + 'instance_type_id': instance_type_id + } + + def test_load_client_json_bridge(self): + mock_config = self.mox.CreateMockAnything() + config = {'client_class': 'JSONBridgeClient', 'client': mock_config} + nova.JSONBridgeClient(mock_config) + self.mox.ReplayAll() + reconciler.Reconciler.load_client(config) + self.mox.VerifyAll() + + def test_load_client_no_class_loads_default_class(self): + mock_config = self.mox.CreateMockAnything() + config = {'client': mock_config} + nova.JSONBridgeClient(mock_config) + self.mox.ReplayAll() + reconciler.Reconciler.load_client(config) + self.mox.VerifyAll() + + def test_load_client_incorrect_class_loads_default_class(self): + mock_config = self.mox.CreateMockAnything() + config = {'client_class': 'BadConfigValue', 'client': mock_config} + nova.JSONBridgeClient(mock_config) + self.mox.ReplayAll() + reconciler.Reconciler.load_client(config) + self.mox.VerifyAll() + + def test_region_for_launch(self): + launch = self.mox.CreateMockAnything() + deployment = self.mox.CreateMockAnything() + deployment.name = 'RegionOne.prod.cell1' + launch.deployment().AndReturn(deployment) + self.mox.ReplayAll() + region = self.reconciler._region_for_launch(launch) + self.assertEqual('RegionOne', region) + self.mox.VerifyAll() + + def test_region_for_launch_no_mapping(self): + launch = self.mox.CreateMockAnything() + deployment = self.mox.CreateMockAnything() + deployment.name = 'RegionOne.prod.cell2' + launch.deployment().AndReturn(deployment) + self.mox.ReplayAll() + region = self.reconciler._region_for_launch(launch) + self.assertFalse(region) + self.mox.VerifyAll() + + def test_region_for_launch_no_raws(self): + launch = self.mox.CreateMockAnything() + launch.deployment() + self.mox.ReplayAll() + region = self.reconciler._region_for_launch(launch) + self.assertFalse(region) + self.mox.VerifyAll() + + def test_missing_exists_for_instance(self): + launch_id = 1 + beginning_d = utils.decimal_utc() + launch = self.mox.CreateMockAnything() + launch.instance = INSTANCE_ID_1 + launch.launched_at = beginning_d - (60*60) + launch.instance_type_id = 1 + models.InstanceUsage.objects.get(id=launch_id).AndReturn(launch) + deployment = self.mox.CreateMockAnything() + launch.deployment().AndReturn(deployment) + deployment.name = 'RegionOne.prod.cell1' + deleted_at = beginning_d - (60*30) + rec_inst = self._fake_reconciler_instance(deleted=True, + deleted_at=deleted_at) + self.client.get_instance('RegionOne', INSTANCE_ID_1).AndReturn(rec_inst) + reconcile_vals = { + 'instance': launch.instance, + 'launched_at': launch.launched_at, + 'deleted_at': deleted_at, + 'instance_type_id': launch.instance_type_id, + 'source': 'reconciler:mocked_client' + } + result = self.mox.CreateMockAnything() + models.InstanceReconcile(**reconcile_vals).AndReturn(result) + result.save() + self.mox.ReplayAll() + result = self.reconciler.missing_exists_for_instance(launch_id, + beginning_d) + self.assertTrue(result) + self.mox.VerifyAll() + + def test_missing_exists_for_instance_not_found(self): + launch_id = 1 + beginning_d = utils.decimal_utc() + launch = self.mox.CreateMockAnything() + launch.instance = INSTANCE_ID_1 + launch.launched_at = beginning_d - (60*60) + launch.instance_type_id = 1 + models.InstanceUsage.objects.get(id=launch_id).AndReturn(launch) + deployment = self.mox.CreateMockAnything() + launch.deployment().AndReturn(deployment) + deployment.name = 'RegionOne.prod.cell1' + ex = exceptions.NotFound() + self.client.get_instance('RegionOne', INSTANCE_ID_1).AndRaise(ex) + self.mox.ReplayAll() + result = self.reconciler.missing_exists_for_instance(launch_id, + beginning_d) + self.assertFalse(result) + self.mox.VerifyAll() + + +json_bridge_config = { + 'url': 'http://json_bridge.example.com/query/', + 'username': 'user', + 'password': 'pass', + 'databases': { + 'RegionOne': 'nova', + } +} + + +class NovaJSONBridgeClientTestCase(unittest.TestCase): + def setUp(self): + self.mox = mox.Mox() + self.client = nova.JSONBridgeClient(json_bridge_config) + self.mox.StubOutWithMock(requests, 'post') + + def tearDown(self): + self.mox.UnsetStubs() + + def mock_for_query(self, database, query, results): + url = json_bridge_config['url'] + database + data = {'sql': query} + auth = (json_bridge_config['username'], json_bridge_config['password']) + result = {'result': results} + response = self.mox.CreateMockAnything() + requests.post(url, data, auth=auth, verify=False)\ + .AndReturn(response) + response.json().AndReturn(result) + + def _fake_instance(self, uuid=INSTANCE_ID_1, launched_at=None, + terminated_at=None, deleted=0, instance_type_id=1): + return { + 'uuid': uuid, + 'launched_at': launched_at, + 'terminated_at': terminated_at, + 'deleted': deleted, + 'instance_type_id': instance_type_id + } + + def test_get_instance(self): + launched_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=5) + launched_at = str(launched_at) + terminated_at = str(datetime.datetime.utcnow()) + results = [self._fake_instance(launched_at=launched_at, + terminated_at=terminated_at, + deleted=True)] + self.mock_for_query('nova', nova.GET_INSTANCE_QUERY % INSTANCE_ID_1, + results) + self.mox.ReplayAll() + instance = self.client.get_instance('RegionOne', INSTANCE_ID_1) + self.assertIsNotNone(instance) + self.assertEqual(instance['id'], INSTANCE_ID_1) + self.assertEqual(instance['instance_type_id'], 1) + launched_at_dec = stackutils.str_time_to_unix(launched_at) + self.assertEqual(instance['launched_at'], launched_at_dec) + terminated_at_dec = stackutils.str_time_to_unix(terminated_at) + self.assertEqual(instance['deleted_at'], terminated_at_dec) + self.assertTrue(instance['deleted']) + self.mox.VerifyAll() + + def test_get_instance_not_found(self): + self.mock_for_query('nova', nova.GET_INSTANCE_QUERY % INSTANCE_ID_1, + []) + self.mox.ReplayAll() + self.assertRaises(exceptions.NotFound, self.client.get_instance, + 'RegionOne', INSTANCE_ID_1) + self.mox.VerifyAll() diff --git a/tests/unit/test_verifier_db.py b/tests/unit/test_verifier_db.py index db99ca6..da75aed 100644 --- a/tests/unit/test_verifier_db.py +++ b/tests/unit/test_verifier_db.py @@ -71,6 +71,9 @@ class VerifierTestCase(unittest.TestCase): self.mox.StubOutWithMock(models, 'InstanceDeletes', use_mock_anything=True) models.InstanceDeletes.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'InstanceReconcile', + use_mock_anything=True) + models.InstanceReconcile.objects = self.mox.CreateMockAnything() self.mox.StubOutWithMock(models, 'InstanceExists', use_mock_anything=True) models.InstanceExists.objects = self.mox.CreateMockAnything() @@ -448,7 +451,8 @@ class VerifierTestCase(unittest.TestCase): dbverifier._verify_for_delete(exist) dbverifier._mark_exist_verified(exist) self.mox.ReplayAll() - dbverifier._verify(exist) + result, exists = dbverifier._verify(exist) + self.assertTrue(result) self.mox.VerifyAll() def test_verify_no_launched_at(self): @@ -460,8 +464,12 @@ class VerifierTestCase(unittest.TestCase): self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') dbverifier._mark_exist_failed(exist, reason="Exists without a launched_at") + self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data') + dbverifier._verify_with_reconciled_data(exist, mox.IgnoreArg())\ + .AndRaise(NotFound('InstanceReconcile', {})) self.mox.ReplayAll() - dbverifier._verify(exist) + result, exists = dbverifier._verify(exist) + self.assertFalse(result) self.mox.VerifyAll() def test_verify_launch_fail(self): @@ -473,9 +481,48 @@ class VerifierTestCase(unittest.TestCase): self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') verify_exception = VerificationException('test') dbverifier._verify_for_launch(exist).AndRaise(verify_exception) + self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data') + dbverifier._verify_with_reconciled_data(exist, verify_exception)\ + .AndRaise(NotFound('InstanceReconcile', {})) dbverifier._mark_exist_failed(exist, reason='test') self.mox.ReplayAll() - dbverifier._verify(exist) + result, exists = dbverifier._verify(exist) + self.assertFalse(result) + self.mox.VerifyAll() + + def test_verify_fail_reconcile_success(self): + exist = self.mox.CreateMockAnything() + exist.launched_at = decimal.Decimal('1.1') + self.mox.StubOutWithMock(dbverifier, '_verify_for_launch') + self.mox.StubOutWithMock(dbverifier, '_verify_for_delete') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') + verify_exception = VerificationException('test') + dbverifier._verify_for_launch(exist).AndRaise(verify_exception) + self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data') + dbverifier._verify_with_reconciled_data(exist, verify_exception) + dbverifier._mark_exist_verified(exist) + self.mox.ReplayAll() + result, exists = dbverifier._verify(exist) + self.assertTrue(result) + self.mox.VerifyAll() + + def test_verify_fail_with_reconciled_data_exception(self): + exist = self.mox.CreateMockAnything() + exist.launched_at = decimal.Decimal('1.1') + self.mox.StubOutWithMock(dbverifier, '_verify_for_launch') + self.mox.StubOutWithMock(dbverifier, '_verify_for_delete') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') + verify_exception = VerificationException('test') + dbverifier._verify_for_launch(exist).AndRaise(verify_exception) + self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data') + dbverifier._verify_with_reconciled_data(exist, verify_exception)\ + .AndRaise(Exception()) + dbverifier._mark_exist_failed(exist, reason='Exception') + self.mox.ReplayAll() + result, exists = dbverifier._verify(exist) + self.assertFalse(result) self.mox.VerifyAll() def test_verify_delete_fail(self): @@ -488,9 +535,13 @@ class VerifierTestCase(unittest.TestCase): verify_exception = VerificationException('test') dbverifier._verify_for_launch(exist) dbverifier._verify_for_delete(exist).AndRaise(verify_exception) + self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data') + dbverifier._verify_with_reconciled_data(exist, verify_exception)\ + .AndRaise(NotFound('InstanceReconcile', {})) dbverifier._mark_exist_failed(exist, reason='test') self.mox.ReplayAll() - dbverifier._verify(exist) + result, exists = dbverifier._verify(exist) + self.assertFalse(result) self.mox.VerifyAll() def test_verify_exception_during_launch(self): @@ -503,7 +554,8 @@ class VerifierTestCase(unittest.TestCase): dbverifier._verify_for_launch(exist).AndRaise(Exception()) dbverifier._mark_exist_failed(exist, reason='Exception') self.mox.ReplayAll() - dbverifier._verify(exist) + result, exists = dbverifier._verify(exist) + self.assertFalse(result) self.mox.VerifyAll() def test_verify_exception_during_delete(self): @@ -517,7 +569,8 @@ class VerifierTestCase(unittest.TestCase): dbverifier._verify_for_delete(exist).AndRaise(Exception()) dbverifier._mark_exist_failed(exist, reason='Exception') self.mox.ReplayAll() - dbverifier._verify(exist) + result, exists = dbverifier._verify(exist) + self.assertFalse(result) self.mox.VerifyAll() def test_verify_for_range_without_callback(self): diff --git a/verifier/dbverifier.py b/verifier/dbverifier.py index 61611c8..9651d6c 100644 --- a/verifier/dbverifier.py +++ b/verifier/dbverifier.py @@ -69,6 +69,15 @@ def _find_launch(instance, launched): return models.InstanceUsage.objects.filter(**params) +def _find_reconcile(instance, launched): + start = launched - datetime.timedelta(microseconds=launched.microsecond) + end = start + datetime.timedelta(microseconds=999999) + params = {'instance': instance, + 'launched_at__gte': dt.dt_to_decimal(start), + 'launched_at__lte': dt.dt_to_decimal(end)} + return models.InstanceReconcile.objects.filter(**params) + + def _find_delete(instance, launched, deleted_max=None): start = launched - datetime.timedelta(microseconds=launched.microsecond) end = start + datetime.timedelta(microseconds=999999) @@ -80,8 +89,16 @@ def _find_delete(instance, launched, deleted_max=None): return models.InstanceDeletes.objects.filter(**params) -def _mark_exist_verified(exist): - exist.status = models.InstanceExists.VERIFIED +def _mark_exist_verified(exist, + reconciled=False, + reason=None): + if not reconciled: + exist.status = models.InstanceExists.VERIFIED + else: + exist.status = models.InstanceExists.RECONCILED + if reason is not None: + exist.fail_reason = reason + exist.save() @@ -152,10 +169,11 @@ def _verify_field_mismatch(exists, launch): launch.os_distro) -def _verify_for_launch(exist): - if exist.usage: +def _verify_for_launch(exist, launch=None, launch_type="InstanceUsage"): + + if not launch and exist.usage: launch = exist.usage - else: + elif not launch: if models.InstanceUsage.objects\ .filter(instance=exist.instance).count() > 0: launches = _find_launch(exist.instance, @@ -166,23 +184,22 @@ def _verify_for_launch(exist): 'launched_at': exist.launched_at } if count > 1: - raise AmbiguousResults('InstanceUsage', query) + raise AmbiguousResults(launch_type, query) elif count == 0: - raise NotFound('InstanceUsage', query) + raise NotFound(launch_type, query) launch = launches[0] else: - raise NotFound('InstanceUsage', {'instance': exist.instance}) + raise NotFound(launch_type, {'instance': exist.instance}) _verify_field_mismatch(exist, launch) -def _verify_for_delete(exist): +def _verify_for_delete(exist, delete=None, delete_type="InstanceDelete"): - delete = None - if exist.delete: + if not delete and exist.delete: # We know we have a delete and we have it's id delete = exist.delete - else: + elif not delete: if exist.deleted_at: # We received this exists before the delete, go find it deletes = _find_delete(exist.instance, @@ -194,7 +211,7 @@ def _verify_for_delete(exist): 'instance': exist.instance, 'launched_at': exist.launched_at } - raise NotFound('InstanceDelete', query) + raise NotFound(delete_type, query) else: # We don't know if this is supposed to have a delete or not. # Thus, we need to check if we have a delete for this instance. @@ -206,7 +223,7 @@ def _verify_for_delete(exist): deleted_at_max = dt.dt_from_decimal(exist.audit_period_ending) deletes = _find_delete(exist.instance, launched_at, deleted_at_max) if deletes.count() > 0: - reason = 'Found InstanceDeletes for non-delete exist' + reason = 'Found %ss for non-delete exist' % delete_type raise VerificationException(reason) if delete: @@ -221,6 +238,31 @@ def _verify_for_delete(exist): delete.deleted_at) +def _verify_with_reconciled_data(exist, ex): + if not exist.launched_at: + raise VerificationException("Exists without a launched_at") + + query = models.InstanceReconcile.objects.filter(instance=exist.instance) + if query.count() > 0: + recs = _find_reconcile(exist.instance, + dt.dt_from_decimal(exist.launched_at)) + search_query = {'instance': exist.instance, + 'launched_at': exist.launched_at} + count = recs.count() + if count > 1: + raise AmbiguousResults('InstanceReconcile', search_query) + elif count == 0: + raise NotFound('InstanceReconcile', search_query) + reconcile = recs[0] + else: + raise NotFound('InstanceReconcile', {'instance': exist.instance}) + + _verify_for_launch(exist, launch=reconcile, + launch_type="InstanceReconcile") + _verify_for_delete(exist, delete=reconcile, + delete_type="InstanceReconcile") + + def _verify(exist): verified = False try: @@ -232,8 +274,23 @@ def _verify(exist): verified = True _mark_exist_verified(exist) - except VerificationException, e: - _mark_exist_failed(exist, reason=str(e)) + except VerificationException, orig_e: + # Something is wrong with the InstanceUsage record + try: + # Attempt to verify against reconciled data + _verify_with_reconciled_data(exist, orig_e) + verified = True + _mark_exist_verified(exist) + except NotFound, rec_e: + # No reconciled data, just mark it failed + _mark_exist_failed(exist, reason=str(orig_e)) + except VerificationException, rec_e: + # Verification failed against reconciled data, mark it failed + # using the second failure. + _mark_exist_failed(exist, reason=str(rec_e)) + except Exception, rec_e: + _mark_exist_failed(exist, reason=rec_e.__class__.__name__) + LOG.exception(rec_e) except Exception, e: _mark_exist_failed(exist, reason=e.__class__.__name__) LOG.exception(e)