x/stacktach
Code Issues Proposed changes

Logging and more obvious match failures

Browse Source
This commit is contained in:
Andrew Melton 2013-07-11 15:08:36 -04:00
parent c3c1412cf2
commit d17e7eef88
5 changed files with 108 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
reports/nova_usage_audit.py
View File

@ -293,7 +293,7 @@ def store_results(start, end, summary, details):
'created': dt.dt_to_decimal(datetime.datetime.utcnow()),
'period_start': start,
'period_end': end,
'version': 3,
'version': 4,
'name': 'nova usage audit'
}

26
stacktach/reconciler/__init__.py
View File

@ -23,6 +23,7 @@ import json
from stacktach import models
from stacktach.reconciler import exceptions
from stacktach.reconciler import nova
from stacktach import stacklog
DEFAULT_CLIENT = nova.JSONBridgeClient
@ -88,7 +89,7 @@ class Reconciler(object):
models.InstanceReconcile(**values).save()
def _fields_match(self, exists, instance):
match = True
match_code = 0
if (exists.launched_at != instance['launched_at'] or
exists.instance_type_id != instance['instance_type_id'] or
@ -97,22 +98,22 @@ class Reconciler(object):
exists.os_distro != instance['os_distro'] or
exists.os_version != instance['os_version'] or
exists.rax_options != instance['rax_options']):
match = False
match_code = 1
if exists.deleted_at is not None:
# Exists says deleted
if (instance['deleted'] and
exists.deleted_at != instance['deleted_at']):
# Nova says deleted, but times don't match
match = False
match_code = 2
elif not instance['deleted']:
# Nova says not deleted
match = False
match_code = 3
elif exists.deleted_at is None and instance['deleted']:
# Exists says not deleted, but Nova says not deleted
match = False
# Exists says not deleted, but Nova says deleted
match_code = 4
return match
return match_code
def missing_exists_for_instance(self, launched_id,
period_beginning):
@ -132,7 +133,7 @@ class Reconciler(object):
deleted_at=instance['deleted_at'])
reconciled = True
except exceptions.NotFound:
reconciled = False
stacklog.info("Couldn't find instance for launch %s" % launched_id)
return reconciled
@ -141,11 +142,16 @@ class Reconciler(object):
region = self._region_for_usage(exists)
try:
instance = self.client.get_instance(region, exists.instance)
if self._fields_match(exists, instance):
match_code = self._fields_match(exists, instance)
if match_code == 0:
self._reconcile_instance(exists, self.client.src_str,
deleted_at=exists.deleted_at)
reconciled = True
else:
msg = "Exists %s failed reconciliation with code %s"
msg %= (exists.id, match_code)
stacklog.info(msg)
except exceptions.NotFound:
pass
stacklog.info("Couldn't find instance for exists %s" % exists.id)
return reconciled

18
stacktach/reconciler/nova.py
View File

@ -5,20 +5,22 @@ from stacktach.reconciler import exceptions
from stacktach.reconciler.utils import empty_reconciler_instance
GET_INSTANCE_QUERY = "SELECT * FROM instances where uuid ='%s';"
GET_INSTANCE_SYSTEM_METADATA = """
SELECT * FROM instance_system_metadata
WHERE instance_uuid = '%s' AND
deleted = 0 AND `key` IN ('image_org.openstack__1__architecture',
'image_org.openstack__1__os_distro',
'image_org.openstack__1__os_version',
'image_com.rackspace__1__options');
"""
METADATA_MAPPING = {
'image_org.openstack__1__architecture': 'os_architecture',
'image_org.openstack__1__os_distro': 'os_distro',
'image_org.openstack__1__os_version': 'os_version',
'image_com.rackspace__1__options': 'rax_options',
}
METADATA_FIELDS = ["'%s'" % x for x in METADATA_MAPPING.keys()]
METADATA_FIELDS = ','.join(METADATA_FIELDS)
GET_INSTANCE_SYSTEM_METADATA = """
SELECT * FROM instance_system_metadata
WHERE instance_uuid = '%s' AND
deleted = 0 AND `key` IN (%s);
"""
GET_INSTANCE_SYSTEM_METADATA %= ('%s', METADATA_FIELDS)
def _json(result):

8
stacktach/stacklog.py
View File

@ -73,4 +73,10 @@ def warn(msg, name=None):
def error(msg, name=None):
if name is None:
name = default_logger_name
get_logger(name=name).error(msg)
get_logger(name=name).error(msg)
def info(msg, name=None):
if name is None:
name = default_logger_name
get_logger(name=name).info(msg)

85
tests/unit/test_reconciler.py
View File

@ -82,7 +82,8 @@ class ReconcilerTestCase(unittest.TestCase):
def tearDown(self):
self.mox.UnsetStubs()
def _fake_usage(self, is_exists=False, is_deleted=False):
def _fake_usage(self, is_exists=False, is_deleted=False,
mock_deployment=False):
usage = self.mox.CreateMockAnything()
usage.id = 1
beginning_d = utils.decimal_utc()
@ -91,17 +92,18 @@ class ReconcilerTestCase(unittest.TestCase):
usage.launched_at = launched_at
usage.instance_type_id = 1
usage.tenant = TENANT_ID_1
if is_exists:
usage.deleted_at = None
if is_deleted:
usage.deleted_at = beginning_d
deployment = self.mox.CreateMockAnything()
deployment.name = 'RegionOne.prod.cell1'
usage.deployment().AndReturn(deployment)
usage.os_architecture = DEFAULT_OS_ARCH
usage.os_distro = DEFAULT_OS_DISTRO
usage.os_version = DEFAULT_OS_VERSION
usage.rax_options = DEFAULT_RAX_OPTIONS
if is_exists:
usage.deleted_at = None
if is_deleted:
usage.deleted_at = beginning_d
if mock_deployment:
deployment = self.mox.CreateMockAnything()
deployment.name = 'RegionOne.prod.cell1'
usage.deployment().AndReturn(deployment)
return usage
def _fake_reconciler_instance(self, uuid=INSTANCE_ID_1, launched_at=None,
@ -179,7 +181,7 @@ class ReconcilerTestCase(unittest.TestCase):
self.mox.VerifyAll()
def test_missing_exists_for_instance(self):
launch = self._fake_usage()
launch = self._fake_usage(mock_deployment=True)
launched_at = launch.launched_at
deleted_at = launched_at + (60*30)
period_beginning = deleted_at + 1
@ -228,7 +230,7 @@ class ReconcilerTestCase(unittest.TestCase):
self.mox.VerifyAll()
def test_failed_validation(self):
exists = self._fake_usage(is_exists=True)
exists = self._fake_usage(is_exists=True, mock_deployment=True)
launched_at = exists.launched_at
rec_inst = self._fake_reconciler_instance(launched_at=launched_at)
self.client.get_instance('RegionOne', INSTANCE_ID_1).AndReturn(rec_inst)
@ -253,7 +255,8 @@ class ReconcilerTestCase(unittest.TestCase):
self.mox.VerifyAll()
def test_failed_validation_deleted(self):
exists = self._fake_usage(is_exists=True, is_deleted=True)
exists = self._fake_usage(is_exists=True, is_deleted=True,
mock_deployment=True)
launched_at = exists.launched_at
deleted_at = exists.deleted_at
rec_inst = self._fake_reconciler_instance(launched_at=launched_at,
@ -336,6 +339,66 @@ class ReconcilerTestCase(unittest.TestCase):
self.assertFalse(result)
self.mox.VerifyAll()
def test_fields_match(self):
exists = self._fake_usage(is_exists=True)
kwargs = {'launched_at': exists.launched_at}
instance = self._fake_reconciler_instance(**kwargs)
self.mox.ReplayAll()
match_code = self.reconciler._fields_match(exists, instance)
self.assertEqual(match_code, 0)
self.mox.VerifyAll()
def test_fields_match_field_with_deleted(self):
exists = self._fake_usage(is_exists=True, is_deleted=True)
kwargs = {'launched_at': exists.launched_at,
'deleted': True,
'deleted_at': exists.deleted_at}
instance = self._fake_reconciler_instance(**kwargs)
self.mox.ReplayAll()
match_code = self.reconciler._fields_match(exists, instance)
self.assertEqual(match_code, 0)
self.mox.VerifyAll()
def test_fields_match_field_miss_match(self):
exists = self._fake_usage(is_exists=True)
kwargs = {'launched_at': exists.launched_at + 1}
instance = self._fake_reconciler_instance(**kwargs)
self.mox.ReplayAll()
match_code = self.reconciler._fields_match(exists, instance)
self.assertEqual(match_code, 1)
self.mox.VerifyAll()
def test_fields_match_field_with_deleted_miss_match(self):
exists = self._fake_usage(is_exists=True, is_deleted=True)
kwargs = {'launched_at': exists.launched_at,
'deleted': True,
'deleted_at': exists.deleted_at+1}
instance = self._fake_reconciler_instance(**kwargs)
self.mox.ReplayAll()
match_code = self.reconciler._fields_match(exists, instance)
self.assertEqual(match_code, 2)
self.mox.VerifyAll()
def test_fields_match_field_not_deleted_in_nova(self):
exists = self._fake_usage(is_exists=True, is_deleted=True)
kwargs = {'launched_at': exists.launched_at}
instance = self._fake_reconciler_instance(**kwargs)
self.mox.ReplayAll()
match_code = self.reconciler._fields_match(exists, instance)
self.assertEqual(match_code, 3)
self.mox.VerifyAll()
def test_fields_match_field_not_deleted_in_exists(self):
exists = self._fake_usage(is_exists=True)
kwargs = {'launched_at': exists.launched_at,
'deleted': True,
'deleted_at': exists.launched_at + 1}
instance = self._fake_reconciler_instance(**kwargs)
self.mox.ReplayAll()
match_code = self.reconciler._fields_match(exists, instance)
self.assertEqual(match_code, 4)
self.mox.VerifyAll()
json_bridge_config = {
'url': 'http://json_bridge.example.com/query/',