x/stacktach
Code Issues Proposed changes

Merge pull request #135 from ramielrowe/reconciler

Browse Source 
Reconciliation in Nova Usage Audit
This commit is contained in:
Andrew Melton 2013-07-01 12:10:14 -07:00
commit e9f2c60320
12 changed files with 824 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
etc/sample_reconciler_config.json Normal file
View File

@ -0,0 +1,13 @@
{
"client_class": "JSONBridgeClient",
"client": {
"url": "http://jsonbridge.example.com:8080/query/",
"username": "bridgeuser",
"password": "super_secure_password",
"regions": {
"RegionOne": "nova-regionone",
"RegionTwo": "nova-regiontwo"
}
},
"region_mapping_loc": "etc/sample_region_mapping.json"
}

6
etc/sample_region_mapping.json Normal file
View File

@ -0,0 +1,6 @@
{
"RegionOne.dev.global": "RegionOne",
"RegionOne.dev.cell1": "RegionOne",
"RegionTwo.dev.global": "RegionTwo",
"RegionTwo.dev.cell1": "RegionTwo"
}

76
reports/nova_usage_audit.py
View File

@ -30,12 +30,21 @@ from django.db.models import F
from stacktach import datetime_to_decimal as dt
from stacktach import models
from stacktach.reconciler import Reconciler
OLD_LAUNCHES_QUERY = "select * from stacktach_instanceusage " \
"where launched_at is not null and " \
"launched_at < %s and instance not in " \
"(select distinct(instance) " \
"from stacktach_instancedeletes where deleted_at < %s)"
OLD_LAUNCHES_QUERY = """
select * from stacktach_instanceusage where
launched_at is not null and
launched_at < %s and
instance not in
(select distinct(instance)
from stacktach_instancedeletes where
deleted_at < %s union
select distinct(instance)
from stacktach_instancereconcile where
deleted_at < %s);"""
reconciler = None
def _get_new_launches(beginning, ending):
@ -63,25 +72,34 @@ def _get_exists(beginning, ending):
return models.InstanceExists.objects.filter(**filters)
def _audit_launches_to_exists(launches, exists):
def _audit_launches_to_exists(launches, exists, beginning):
fails = []
for (instance, launches) in launches.items():
if instance in exists:
for launch1 in launches:
for expected in launches:
found = False
for launch2 in exists[instance]:
if int(launch1['launched_at']) == int(launch2['launched_at']):
for actual in exists[instance]:
if int(expected['launched_at']) == \
int(actual['launched_at']):
# HACK (apmelton): Truncate the decimal because we may not
# have the milliseconds.
found = True
if not found:
rec = False
if reconciler:
args = (expected['id'], beginning)
rec = reconciler.missing_exists_for_instance(*args)
msg = "Couldn't find exists for launch (%s, %s)"
msg = msg % (instance, launch1['launched_at'])
fails.append(['Launch', launch1['id'], msg])
msg = msg % (instance, expected['launched_at'])
fails.append(['Launch', expected['id'], msg, 'Y' if rec else 'N'])
else:
rec = False
if reconciler:
args = (launches[0]['id'], beginning)
rec = reconciler.missing_exists_for_instance(*args)
msg = "No exists for instance (%s)" % instance
fails.append(['Launch', '-', msg])
fails.append(['Launch', '-', msg, 'Y' if rec else 'N'])
return fails
@ -175,8 +193,13 @@ def _launch_audit_for_period(beginning, ending):
else:
launches_dict[instance] = [l, ]
old_launches = models.InstanceUsage.objects.raw(OLD_LAUNCHES_QUERY,
[beginning, beginning])
# NOTE (apmelton)
# Django's safe substitution doesn't allow dict substitution...
# Thus, we send it 'beginning' three times...
old_launches = models.InstanceUsage.objects\
.raw(OLD_LAUNCHES_QUERY,
[beginning, beginning, beginning])
old_launches_dict = {}
for launch in old_launches:
instance = launch.instance
@ -205,7 +228,8 @@ def _launch_audit_for_period(beginning, ending):
exists_dict[instance] = [e, ]
launch_to_exists_fails = _audit_launches_to_exists(launches_dict,
exists_dict)
exists_dict,
beginning)
return launch_to_exists_fails, new_launches.count(), len(old_launches_dict)
@ -222,11 +246,11 @@ def audit_for_period(beginning, ending):
summary = {
'verifier': verify_summary,
'launch_fails': {
'total_failures': len(detail),
'launch_summary': {
'new_launches': new_count,
'old_launches': old_count
}
'old_launches': old_count,
'failures': len(detail)
},
}
details = {
@ -276,7 +300,7 @@ def store_results(start, end, summary, details):
def make_json_report(summary, details):
report = [{'summary': summary},
['Object', 'ID', 'Error Description']]
['Object', 'ID', 'Error Description', 'Reconciled?']]
report.extend(details['exist_fails'])
report.extend(details['launch_fails'])
return json.dumps(report)
@ -302,8 +326,20 @@ if __name__ == '__main__':
help="If set to true, report will be stored. "
"Otherwise, it will just be printed",
type=bool, default=False)
parser.add_argument('--reconcile',
help="Enabled reconciliation",
type=bool, default=False)
parser.add_argument('--reconciler_config',
help="Location of the reconciler config file",
type=str,
default='/etc/stacktach/reconciler-config.json')
args = parser.parse_args()
if args.reconcile:
with open(args.reconciler_config) as f:
reconciler_config = json.load(f)
reconciler = Reconciler(reconciler_config)
if args.utcdatetime is not None:
time = args.utcdatetime
else:

160
stacktach/migrations/0004_create_instancereconcile.py Normal file
View File

@ -0,0 +1,160 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding model 'InstanceReconcile'
db.create_table(u'stacktach_instancereconcile', (
(u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('row_created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
('row_updated', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)),
('instance', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)),
('launched_at', self.gf('django.db.models.fields.DecimalField')(null=True, max_digits=20, decimal_places=6, db_index=True)),
('deleted_at', self.gf('django.db.models.fields.DecimalField')(null=True, max_digits=20, decimal_places=6, db_index=True)),
('instance_type_id', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)),
('source', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=150, null=True, blank=True)),
))
db.send_create_signal(u'stacktach', ['InstanceReconcile'])
def backwards(self, orm):
# Deleting model 'InstanceReconcile'
db.delete_table(u'stacktach_instancereconcile')
models = {
u'stacktach.deployment': {
'Meta': {'object_name': 'Deployment'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
},
u'stacktach.instancedeletes': {
'Meta': {'object_name': 'InstanceDeletes'},
'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
'raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.RawData']", 'null': 'True'})
},
u'stacktach.instanceexists': {
'Meta': {'object_name': 'InstanceExists'},
'audit_period_beginning': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
'audit_period_ending': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
'delete': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.InstanceDeletes']"}),
'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
'fail_reason': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '300', 'null': 'True', 'blank': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'instance_type_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
'message_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'os_architecture': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'os_distro': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'os_version': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'raw': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.RawData']"}),
'rax_options': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'send_status': ('django.db.models.fields.IntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True'}),
'status': ('django.db.models.fields.CharField', [], {'default': "'pending'", 'max_length': '50', 'db_index': 'True'}),
'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'usage': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.InstanceUsage']"})
},
u'stacktach.instancereconcile': {
'Meta': {'object_name': 'InstanceReconcile'},
'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'instance_type_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
'row_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'row_updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'source': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '150', 'null': 'True', 'blank': 'True'})
},
u'stacktach.instanceusage': {
'Meta': {'object_name': 'InstanceUsage'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'instance_type_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
'os_architecture': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'os_distro': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'os_version': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'rax_options': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'request_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'})
},
u'stacktach.jsonreport': {
'Meta': {'object_name': 'JsonReport'},
'created': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'json': ('django.db.models.fields.TextField', [], {}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
'period_end': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
'period_start': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
'version': ('django.db.models.fields.IntegerField', [], {'default': '1'})
},
u'stacktach.lifecycle': {
'Meta': {'object_name': 'Lifecycle'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'last_raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.RawData']", 'null': 'True'}),
'last_state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'last_task_state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'})
},
u'stacktach.rawdata': {
'Meta': {'object_name': 'RawData'},
'deployment': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Deployment']"}),
'event': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'host': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'image_type': ('django.db.models.fields.IntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True'}),
'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'json': ('django.db.models.fields.TextField', [], {}),
'old_state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '20', 'null': 'True', 'blank': 'True'}),
'old_task': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '30', 'null': 'True', 'blank': 'True'}),
'publisher': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}),
'request_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'routing_key': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'service': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '20', 'null': 'True', 'blank': 'True'}),
'task': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '30', 'null': 'True', 'blank': 'True'}),
'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'when': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'})
},
u'stacktach.rawdataimagemeta': {
'Meta': {'object_name': 'RawDataImageMeta'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'os_architecture': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'os_distro': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'os_version': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.RawData']"}),
'rax_options': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'})
},
u'stacktach.requesttracker': {
'Meta': {'object_name': 'RequestTracker'},
'completed': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
'duration': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'last_timing': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Timing']", 'null': 'True'}),
'lifecycle': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Lifecycle']"}),
'request_id': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
'start': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'})
},
u'stacktach.timing': {
'Meta': {'object_name': 'Timing'},
'diff': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
'end_raw': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.RawData']"}),
'end_when': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'lifecycle': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Lifecycle']"}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
'start_raw': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.RawData']"}),
'start_when': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6'})
}
}
complete_apps = ['stacktach']

26
stacktach/models.py
View File

@ -101,6 +101,13 @@ class InstanceUsage(models.Model):
os_version = models.TextField(null=True, blank=True)
rax_options = models.TextField(null=True, blank=True)
def deployment(self):
raws = RawData.objects.filter(request_id=self.request_id)
if raws.count() == 0:
return False
raw = raws[0]
return raw.deployment
class InstanceDeletes(models.Model):
instance = models.CharField(max_length=50, null=True,
blank=True, db_index=True)
@ -111,15 +118,34 @@ class InstanceDeletes(models.Model):
raw = models.ForeignKey(RawData, null=True)
class InstanceReconcile(models.Model):
row_created = models.DateTimeField(auto_now_add=True)
row_updated = models.DateTimeField(auto_now=True)
instance = models.CharField(max_length=50, null=True,
blank=True, db_index=True)
launched_at = models.DecimalField(null=True, max_digits=20,
decimal_places=6, db_index=True)
deleted_at = models.DecimalField(null=True, max_digits=20,
decimal_places=6, db_index=True)
instance_type_id = models.CharField(max_length=50,
null=True,
blank=True,
db_index=True)
source = models.CharField(max_length=150, null=True,
blank=True, db_index=True)
class InstanceExists(models.Model):
PENDING = 'pending'
VERIFYING = 'verifying'
VERIFIED = 'verified'
RECONCILED = 'reconciled'
FAILED = 'failed'
STATUS_CHOICES = [
(PENDING, 'Pending Verification'),
(VERIFYING, 'Currently Being Verified'),
(VERIFIED, 'Passed Verification'),
(RECONCILED, 'Passed Verification After Reconciliation'),
(FAILED, 'Failed Verification'),
]
instance = models.CharField(max_length=50, null=True,

107
stacktach/reconciler/__init__.py Normal file
View File

@ -0,0 +1,107 @@
# Copyright (c) 2013 - Rackspace Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
import json
from stacktach import models
from stacktach.reconciler import exceptions
from stacktach.reconciler import nova
DEFAULT_CLIENT = nova.JSONBridgeClient
CONFIG = {
'client_class': 'JSONBridgeClient',
'client': {
'url': 'http://stack.dev.ramielrowe.com:8080/query/',
'username': '',
'password': '',
'databases': {
'RegionOne': 'nova',
}
},
'region_mapping_loc': '/etc/stacktach/region_mapping.json'
}
class Reconciler(object):
def __init__(self, config, client=None, region_mapping=None):
self.config = config
self.client = (client or Reconciler.load_client(config))
self.region_mapping = (region_mapping or
Reconciler.load_region_mapping(config))
@classmethod
def load_client(cls, config):
client_class = config.get('client_class')
if client_class == 'JSONBridgeClient':
return nova.JSONBridgeClient(config['client'])
else:
return DEFAULT_CLIENT(config['client'])
@classmethod
def load_region_mapping(cls, config):
with open(config['region_mapping_loc']) as f:
return json.load(f)
def _region_for_launch(self, launch):
deployment = launch.deployment()
if deployment:
deployment_name = str(deployment.name)
if deployment_name in self.region_mapping:
return self.region_mapping[deployment_name]
else:
return False
else:
return False
def _reconcile_instance(self, launch, src,
launched_at=None, deleted_at=None,
instance_type_id=None):
values = {
'instance': launch.instance,
'launched_at': (launched_at or launch.launched_at),
'deleted_at': deleted_at,
'instance_type_id': (instance_type_id or launch.instance_type_id),
'source': 'reconciler:%s' % src,
}
models.InstanceReconcile(**values).save()
def missing_exists_for_instance(self, launched_id,
period_beginning):
reconciled = False
launch = models.InstanceUsage.objects.get(id=launched_id)
region = self._region_for_launch(launch)
try:
instance = self.client.get_instance(region, launch.instance)
if instance['deleted'] and instance['deleted_at'] is not None:
# Check to see if instance has been deleted
deleted_at = instance['deleted_at']
if deleted_at < period_beginning:
# Check to see if instance was deleted before period.
# If so, we shouldn't expect an exists.
self._reconcile_instance(launch, self.client.src_str,
deleted_at=instance['deleted_at'])
reconciled = True
except exceptions.NotFound:
reconciled = False
return reconciled

3
stacktach/reconciler/exceptions.py Normal file
View File

@ -0,0 +1,3 @@
class NotFound(Exception):
def __init__(self, message="NotFound"):
self.message = message

54
stacktach/reconciler/nova.py Normal file
View File

@ -0,0 +1,54 @@
import json
import requests
from stacktach import utils as stackutils
from stacktach.reconciler import exceptions
from stacktach.reconciler.utils import empty_reconciler_instance
GET_INSTANCE_QUERY = "SELECT * FROM instances where uuid ='%s';"
class JSONBridgeClient(object):
src_str = 'json_bridge:nova_db'
def __init__(self, config):
self.config = config
def _url_for_region(self, region):
return self.config['url'] + self.config['databases'][region]
def _do_query(self, region, query):
data = {'sql': query}
credentials = (self.config['username'], self.config['password'])
return requests.post(self._url_for_region(region), data,
verify=False, auth=credentials).json()
def _to_reconciler_instance(self, instance):
r_instance = empty_reconciler_instance()
r_instance.update({
'id': instance['uuid'],
'instance_type_id': instance['instance_type_id'],
})
if instance['launched_at'] is not None:
launched_at = stackutils.str_time_to_unix(instance['launched_at'])
r_instance['launched_at'] = launched_at
if instance['terminated_at'] is not None:
deleted_at = stackutils.str_time_to_unix(instance['terminated_at'])
r_instance['deleted_at'] = deleted_at
if instance['deleted'] != 0:
r_instance['deleted'] = True
return r_instance
def get_instance(self, region, uuid):
results = self._do_query(region, GET_INSTANCE_QUERY % uuid)['result']
if len(results) > 0:
return self._to_reconciler_instance(results[0])
else:
msg = "Couldn't find instance (%s) using JSON Bridge in region (%s)"
raise exceptions.NotFound(msg % (uuid, region))

9
stacktach/reconciler/utils.py Normal file
View File

@ -0,0 +1,9 @@
def empty_reconciler_instance():
r_instance = {
'id': None,
'launched_at': None,
'deleted': False,
'deleted_at': None,
'instance_type_ud': None
}
return r_instance

258
tests/unit/test_reconciler.py Normal file
View File

@ -0,0 +1,258 @@
# Copyright (c) 2013 - Rackspace Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
import datetime
import unittest
import mox
import requests
from stacktach import models
from stacktach import reconciler
from stacktach import utils as stackutils
from stacktach.reconciler import exceptions
from stacktach.reconciler import nova
from tests.unit import utils
from tests.unit.utils import INSTANCE_ID_1
region_mapping = {
'RegionOne.prod.cell1': 'RegionOne',
'RegionTwo.prod.cell1': 'RegionTwo',
}
class ReconcilerTestCase(unittest.TestCase):
def setUp(self):
self.mox = mox.Mox()
self.client = self.mox.CreateMockAnything()
self.client.src_str = 'mocked_client'
self.reconciler = reconciler.Reconciler({},
client=self.client,
region_mapping=region_mapping)
self.mox.StubOutWithMock(models, 'RawData', use_mock_anything=True)
models.RawData.objects = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(models, 'Deployment', use_mock_anything=True)
models.Deployment.objects = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(models, 'Lifecycle', use_mock_anything=True)
models.Lifecycle.objects = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(models, 'Timing', use_mock_anything=True)
models.Timing.objects = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(models, 'RequestTracker',
use_mock_anything=True)
models.RequestTracker.objects = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(models, 'InstanceUsage',
use_mock_anything=True)
models.InstanceUsage.objects = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(models, 'InstanceReconcile',
use_mock_anything=True)
models.InstanceReconcile.objects = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(models, 'InstanceDeletes',
use_mock_anything=True)
models.InstanceDeletes.objects = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(models, 'InstanceExists',
use_mock_anything=True)
models.InstanceExists.objects = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(models, 'JsonReport', use_mock_anything=True)
models.JsonReport.objects = self.mox.CreateMockAnything()
def tearDown(self):
self.mox.UnsetStubs()
def _fake_reconciler_instance(self, uuid=INSTANCE_ID_1, launched_at=None,
deleted_at=None, deleted=False,
instance_type_id=1):
return {
'id': uuid,
'launched_at': launched_at,
'deleted_at': deleted_at,
'deleted': deleted,
'instance_type_id': instance_type_id
}
def test_load_client_json_bridge(self):
mock_config = self.mox.CreateMockAnything()
config = {'client_class': 'JSONBridgeClient', 'client': mock_config}
nova.JSONBridgeClient(mock_config)
self.mox.ReplayAll()
reconciler.Reconciler.load_client(config)
self.mox.VerifyAll()
def test_load_client_no_class_loads_default_class(self):
mock_config = self.mox.CreateMockAnything()
config = {'client': mock_config}
nova.JSONBridgeClient(mock_config)
self.mox.ReplayAll()
reconciler.Reconciler.load_client(config)
self.mox.VerifyAll()
def test_load_client_incorrect_class_loads_default_class(self):
mock_config = self.mox.CreateMockAnything()
config = {'client_class': 'BadConfigValue', 'client': mock_config}
nova.JSONBridgeClient(mock_config)
self.mox.ReplayAll()
reconciler.Reconciler.load_client(config)
self.mox.VerifyAll()
def test_region_for_launch(self):
launch = self.mox.CreateMockAnything()
deployment = self.mox.CreateMockAnything()
deployment.name = 'RegionOne.prod.cell1'
launch.deployment().AndReturn(deployment)
self.mox.ReplayAll()
region = self.reconciler._region_for_launch(launch)
self.assertEqual('RegionOne', region)
self.mox.VerifyAll()
def test_region_for_launch_no_mapping(self):
launch = self.mox.CreateMockAnything()
deployment = self.mox.CreateMockAnything()
deployment.name = 'RegionOne.prod.cell2'
launch.deployment().AndReturn(deployment)
self.mox.ReplayAll()
region = self.reconciler._region_for_launch(launch)
self.assertFalse(region)
self.mox.VerifyAll()
def test_region_for_launch_no_raws(self):
launch = self.mox.CreateMockAnything()
launch.deployment()
self.mox.ReplayAll()
region = self.reconciler._region_for_launch(launch)
self.assertFalse(region)
self.mox.VerifyAll()
def test_missing_exists_for_instance(self):
launch_id = 1
beginning_d = utils.decimal_utc()
launch = self.mox.CreateMockAnything()
launch.instance = INSTANCE_ID_1
launch.launched_at = beginning_d - (60*60)
launch.instance_type_id = 1
models.InstanceUsage.objects.get(id=launch_id).AndReturn(launch)
deployment = self.mox.CreateMockAnything()
launch.deployment().AndReturn(deployment)
deployment.name = 'RegionOne.prod.cell1'
deleted_at = beginning_d - (60*30)
rec_inst = self._fake_reconciler_instance(deleted=True,
deleted_at=deleted_at)
self.client.get_instance('RegionOne', INSTANCE_ID_1).AndReturn(rec_inst)
reconcile_vals = {
'instance': launch.instance,
'launched_at': launch.launched_at,
'deleted_at': deleted_at,
'instance_type_id': launch.instance_type_id,
'source': 'reconciler:mocked_client'
}
result = self.mox.CreateMockAnything()
models.InstanceReconcile(**reconcile_vals).AndReturn(result)
result.save()
self.mox.ReplayAll()
result = self.reconciler.missing_exists_for_instance(launch_id,
beginning_d)
self.assertTrue(result)
self.mox.VerifyAll()
def test_missing_exists_for_instance_not_found(self):
launch_id = 1
beginning_d = utils.decimal_utc()
launch = self.mox.CreateMockAnything()
launch.instance = INSTANCE_ID_1
launch.launched_at = beginning_d - (60*60)
launch.instance_type_id = 1
models.InstanceUsage.objects.get(id=launch_id).AndReturn(launch)
deployment = self.mox.CreateMockAnything()
launch.deployment().AndReturn(deployment)
deployment.name = 'RegionOne.prod.cell1'
ex = exceptions.NotFound()
self.client.get_instance('RegionOne', INSTANCE_ID_1).AndRaise(ex)
self.mox.ReplayAll()
result = self.reconciler.missing_exists_for_instance(launch_id,
beginning_d)
self.assertFalse(result)
self.mox.VerifyAll()
json_bridge_config = {
'url': 'http://json_bridge.example.com/query/',
'username': 'user',
'password': 'pass',
'databases': {
'RegionOne': 'nova',
}
}
class NovaJSONBridgeClientTestCase(unittest.TestCase):
def setUp(self):
self.mox = mox.Mox()
self.client = nova.JSONBridgeClient(json_bridge_config)
self.mox.StubOutWithMock(requests, 'post')
def tearDown(self):
self.mox.UnsetStubs()
def mock_for_query(self, database, query, results):
url = json_bridge_config['url'] + database
data = {'sql': query}
auth = (json_bridge_config['username'], json_bridge_config['password'])
result = {'result': results}
response = self.mox.CreateMockAnything()
requests.post(url, data, auth=auth, verify=False)\
.AndReturn(response)
response.json().AndReturn(result)
def _fake_instance(self, uuid=INSTANCE_ID_1, launched_at=None,
terminated_at=None, deleted=0, instance_type_id=1):
return {
'uuid': uuid,
'launched_at': launched_at,
'terminated_at': terminated_at,
'deleted': deleted,
'instance_type_id': instance_type_id
}
def test_get_instance(self):
launched_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=5)
launched_at = str(launched_at)
terminated_at = str(datetime.datetime.utcnow())
results = [self._fake_instance(launched_at=launched_at,
terminated_at=terminated_at,
deleted=True)]
self.mock_for_query('nova', nova.GET_INSTANCE_QUERY % INSTANCE_ID_1,
results)
self.mox.ReplayAll()
instance = self.client.get_instance('RegionOne', INSTANCE_ID_1)
self.assertIsNotNone(instance)
self.assertEqual(instance['id'], INSTANCE_ID_1)
self.assertEqual(instance['instance_type_id'], 1)
launched_at_dec = stackutils.str_time_to_unix(launched_at)
self.assertEqual(instance['launched_at'], launched_at_dec)
terminated_at_dec = stackutils.str_time_to_unix(terminated_at)
self.assertEqual(instance['deleted_at'], terminated_at_dec)
self.assertTrue(instance['deleted'])
self.mox.VerifyAll()
def test_get_instance_not_found(self):
self.mock_for_query('nova', nova.GET_INSTANCE_QUERY % INSTANCE_ID_1,
[])
self.mox.ReplayAll()
self.assertRaises(exceptions.NotFound, self.client.get_instance,
'RegionOne', INSTANCE_ID_1)
self.mox.VerifyAll()

65
tests/unit/test_verifier_db.py
View File

@ -71,6 +71,9 @@ class VerifierTestCase(unittest.TestCase):
self.mox.StubOutWithMock(models, 'InstanceDeletes',
use_mock_anything=True)
models.InstanceDeletes.objects = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(models, 'InstanceReconcile',
use_mock_anything=True)
models.InstanceReconcile.objects = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(models, 'InstanceExists',
use_mock_anything=True)
models.InstanceExists.objects = self.mox.CreateMockAnything()
@ -448,7 +451,8 @@ class VerifierTestCase(unittest.TestCase):
dbverifier._verify_for_delete(exist)
dbverifier._mark_exist_verified(exist)
self.mox.ReplayAll()
dbverifier._verify(exist)
result, exists = dbverifier._verify(exist)
self.assertTrue(result)
self.mox.VerifyAll()
def test_verify_no_launched_at(self):
@ -460,8 +464,12 @@ class VerifierTestCase(unittest.TestCase):
self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified')
dbverifier._mark_exist_failed(exist,
reason="Exists without a launched_at")
self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data')
dbverifier._verify_with_reconciled_data(exist, mox.IgnoreArg())\
.AndRaise(NotFound('InstanceReconcile', {}))
self.mox.ReplayAll()
dbverifier._verify(exist)
result, exists = dbverifier._verify(exist)
self.assertFalse(result)
self.mox.VerifyAll()
def test_verify_launch_fail(self):
@ -473,9 +481,48 @@ class VerifierTestCase(unittest.TestCase):
self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified')
verify_exception = VerificationException('test')
dbverifier._verify_for_launch(exist).AndRaise(verify_exception)
self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data')
dbverifier._verify_with_reconciled_data(exist, verify_exception)\
.AndRaise(NotFound('InstanceReconcile', {}))
dbverifier._mark_exist_failed(exist, reason='test')
self.mox.ReplayAll()
dbverifier._verify(exist)
result, exists = dbverifier._verify(exist)
self.assertFalse(result)
self.mox.VerifyAll()
def test_verify_fail_reconcile_success(self):
exist = self.mox.CreateMockAnything()
exist.launched_at = decimal.Decimal('1.1')
self.mox.StubOutWithMock(dbverifier, '_verify_for_launch')
self.mox.StubOutWithMock(dbverifier, '_verify_for_delete')
self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed')
self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified')
verify_exception = VerificationException('test')
dbverifier._verify_for_launch(exist).AndRaise(verify_exception)
self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data')
dbverifier._verify_with_reconciled_data(exist, verify_exception)
dbverifier._mark_exist_verified(exist)
self.mox.ReplayAll()
result, exists = dbverifier._verify(exist)
self.assertTrue(result)
self.mox.VerifyAll()
def test_verify_fail_with_reconciled_data_exception(self):
exist = self.mox.CreateMockAnything()
exist.launched_at = decimal.Decimal('1.1')
self.mox.StubOutWithMock(dbverifier, '_verify_for_launch')
self.mox.StubOutWithMock(dbverifier, '_verify_for_delete')
self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed')
self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified')
verify_exception = VerificationException('test')
dbverifier._verify_for_launch(exist).AndRaise(verify_exception)
self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data')
dbverifier._verify_with_reconciled_data(exist, verify_exception)\
.AndRaise(Exception())
dbverifier._mark_exist_failed(exist, reason='Exception')
self.mox.ReplayAll()
result, exists = dbverifier._verify(exist)
self.assertFalse(result)
self.mox.VerifyAll()
def test_verify_delete_fail(self):
@ -488,9 +535,13 @@ class VerifierTestCase(unittest.TestCase):
verify_exception = VerificationException('test')
dbverifier._verify_for_launch(exist)
dbverifier._verify_for_delete(exist).AndRaise(verify_exception)
self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data')
dbverifier._verify_with_reconciled_data(exist, verify_exception)\
.AndRaise(NotFound('InstanceReconcile', {}))
dbverifier._mark_exist_failed(exist, reason='test')
self.mox.ReplayAll()
dbverifier._verify(exist)
result, exists = dbverifier._verify(exist)
self.assertFalse(result)
self.mox.VerifyAll()
def test_verify_exception_during_launch(self):
@ -503,7 +554,8 @@ class VerifierTestCase(unittest.TestCase):
dbverifier._verify_for_launch(exist).AndRaise(Exception())
dbverifier._mark_exist_failed(exist, reason='Exception')
self.mox.ReplayAll()
dbverifier._verify(exist)
result, exists = dbverifier._verify(exist)
self.assertFalse(result)
self.mox.VerifyAll()
def test_verify_exception_during_delete(self):
@ -517,7 +569,8 @@ class VerifierTestCase(unittest.TestCase):
dbverifier._verify_for_delete(exist).AndRaise(Exception())
dbverifier._mark_exist_failed(exist, reason='Exception')
self.mox.ReplayAll()
dbverifier._verify(exist)
result, exists = dbverifier._verify(exist)
self.assertFalse(result)
self.mox.VerifyAll()
def test_verify_for_range_without_callback(self):

87
verifier/dbverifier.py
View File

@ -69,6 +69,15 @@ def _find_launch(instance, launched):
return models.InstanceUsage.objects.filter(**params)
def _find_reconcile(instance, launched):
start = launched - datetime.timedelta(microseconds=launched.microsecond)
end = start + datetime.timedelta(microseconds=999999)
params = {'instance': instance,
'launched_at__gte': dt.dt_to_decimal(start),
'launched_at__lte': dt.dt_to_decimal(end)}
return models.InstanceReconcile.objects.filter(**params)
def _find_delete(instance, launched, deleted_max=None):
start = launched - datetime.timedelta(microseconds=launched.microsecond)
end = start + datetime.timedelta(microseconds=999999)
@ -80,8 +89,16 @@ def _find_delete(instance, launched, deleted_max=None):
return models.InstanceDeletes.objects.filter(**params)
def _mark_exist_verified(exist):
def _mark_exist_verified(exist,
reconciled=False,
reason=None):
if not reconciled:
exist.status = models.InstanceExists.VERIFIED
else:
exist.status = models.InstanceExists.RECONCILED
if reason is not None:
exist.fail_reason = reason
exist.save()
@ -152,10 +169,11 @@ def _verify_field_mismatch(exists, launch):
launch.os_distro)
def _verify_for_launch(exist):
if exist.usage:
def _verify_for_launch(exist, launch=None, launch_type="InstanceUsage"):
if not launch and exist.usage:
launch = exist.usage
else:
elif not launch:
if models.InstanceUsage.objects\
.filter(instance=exist.instance).count() > 0:
launches = _find_launch(exist.instance,
@ -166,23 +184,22 @@ def _verify_for_launch(exist):
'launched_at': exist.launched_at
}
if count > 1:
raise AmbiguousResults('InstanceUsage', query)
raise AmbiguousResults(launch_type, query)
elif count == 0:
raise NotFound('InstanceUsage', query)
raise NotFound(launch_type, query)
launch = launches[0]
else:
raise NotFound('InstanceUsage', {'instance': exist.instance})
raise NotFound(launch_type, {'instance': exist.instance})
_verify_field_mismatch(exist, launch)
def _verify_for_delete(exist):
def _verify_for_delete(exist, delete=None, delete_type="InstanceDelete"):
delete = None
if exist.delete:
if not delete and exist.delete:
# We know we have a delete and we have it's id
delete = exist.delete
else:
elif not delete:
if exist.deleted_at:
# We received this exists before the delete, go find it
deletes = _find_delete(exist.instance,
@ -194,7 +211,7 @@ def _verify_for_delete(exist):
'instance': exist.instance,
'launched_at': exist.launched_at
}
raise NotFound('InstanceDelete', query)
raise NotFound(delete_type, query)
else:
# We don't know if this is supposed to have a delete or not.
# Thus, we need to check if we have a delete for this instance.
@ -206,7 +223,7 @@ def _verify_for_delete(exist):
deleted_at_max = dt.dt_from_decimal(exist.audit_period_ending)
deletes = _find_delete(exist.instance, launched_at, deleted_at_max)
if deletes.count() > 0:
reason = 'Found InstanceDeletes for non-delete exist'
reason = 'Found %ss for non-delete exist' % delete_type
raise VerificationException(reason)
if delete:
@ -221,6 +238,31 @@ def _verify_for_delete(exist):
delete.deleted_at)
def _verify_with_reconciled_data(exist, ex):
if not exist.launched_at:
raise VerificationException("Exists without a launched_at")
query = models.InstanceReconcile.objects.filter(instance=exist.instance)
if query.count() > 0:
recs = _find_reconcile(exist.instance,
dt.dt_from_decimal(exist.launched_at))
search_query = {'instance': exist.instance,
'launched_at': exist.launched_at}
count = recs.count()
if count > 1:
raise AmbiguousResults('InstanceReconcile', search_query)
elif count == 0:
raise NotFound('InstanceReconcile', search_query)
reconcile = recs[0]
else:
raise NotFound('InstanceReconcile', {'instance': exist.instance})
_verify_for_launch(exist, launch=reconcile,
launch_type="InstanceReconcile")
_verify_for_delete(exist, delete=reconcile,
delete_type="InstanceReconcile")
def _verify(exist):
verified = False
try:
@ -232,8 +274,23 @@ def _verify(exist):
verified = True
_mark_exist_verified(exist)
except VerificationException, e:
_mark_exist_failed(exist, reason=str(e))
except VerificationException, orig_e:
# Something is wrong with the InstanceUsage record
try:
# Attempt to verify against reconciled data
_verify_with_reconciled_data(exist, orig_e)
verified = True
_mark_exist_verified(exist)
except NotFound, rec_e:
# No reconciled data, just mark it failed
_mark_exist_failed(exist, reason=str(orig_e))
except VerificationException, rec_e:
# Verification failed against reconciled data, mark it failed
# using the second failure.
_mark_exist_failed(exist, reason=str(rec_e))
except Exception, rec_e:
_mark_exist_failed(exist, reason=rec_e.__class__.__name__)
LOG.exception(rec_e)
except Exception, e:
_mark_exist_failed(exist, reason=e.__class__.__name__)
LOG.exception(e)