From ccc7e527f63d8bb0c2472592f5ad71cb2fb94243 Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Thu, 25 Apr 2013 13:59:55 -0400 Subject: [PATCH 01/10] Starting reconciler --- stacktach/models.py | 17 +++++++++++ stacktach/reconciler.py | 67 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 stacktach/reconciler.py diff --git a/stacktach/models.py b/stacktach/models.py index 6150d6f..6236695 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -99,6 +99,23 @@ class InstanceDeletes(models.Model): raw = models.ForeignKey(RawData, null=True) +class InstanceReconcile(models.Model): + row_created = models.DateTimeField(auto_now_add=True) + row_updated = models.DateTimeField(auto_now=True) + instance = models.CharField(max_length=50, null=True, + blank=True, db_index=True) + launched_at = models.DecimalField(null=True, max_digits=20, + decimal_places=6, db_index=True) + deleted_at = models.DecimalField(null=True, max_digits=20, + decimal_places=6, db_index=True) + instance_type_id = models.CharField(max_length=50, + null=True, + blank=True, + db_index=True) + source = models.CharField(max_length=150, null=True, + blank=True, db_index=True) + + class InstanceExists(models.Model): PENDING = 'pending' VERIFYING = 'verifying' diff --git a/stacktach/reconciler.py b/stacktach/reconciler.py new file mode 100644 index 0000000..c9e18e4 --- /dev/null +++ b/stacktach/reconciler.py @@ -0,0 +1,67 @@ +from novaclient.v1_1 import client + +from stacktach import models + +reconciler_config = { + 'nova':{ + 'DFW':{ + 'username': 'm0lt3n', + 'project_id': '724740', + 'api_key': '', + 'auth_url': 'https://identity.api.rackspacecloud.com/v2.0', + 'auth_system': 'rackspace', + }, + 'ORD':{ + 'username': 'm0lt3n', + 'project_id': '724740', + 'api_key': '', + 'auth_url': 'https://identity.api.rackspacecloud.com/v2.0', + 'auth_system': 'rackspace', + }, + + }, + 'region_mapping_loc': '/etc/stacktach/region_mapping.json' +} + +region_mapping = { + 'x': 'DFW' +} + + +class Reconciler(object): + + def __init__(self, config): + self.config = reconciler_config + self.region_mapping = region_mapping + self.nova_clients = {} + + def _get_nova(self, region): + if region in self.nova_clients: + return self.nova_clients[region] + + region_cfg = self.config['nova'][region] + region_auth_system = region_cfg.get('auth_system', 'keystone') + + nova = client.Client(region_cfg['username'], region_cfg['api_key'], + region_cfg['project_id'], + auth_url=region_cfg['auth_url'], + auth_system=region_auth_system) + + self.nova_clients[region] = nova + return nova + + def _region_for_launch(self, launch): + request = launch.request_id + raws = models.RawData.objects.filter(request_id=request) + if raws.count() == 0: + return False + raw = raws[0] + return self.region_mapping[str(raw.deployment.name)] + + def missing_exists_for_instance(self, launched_id, + period_beginning, + period_ending): + launch = models.InstanceUsage.objects.get(id=launched_id) + region = self._region_for_launch(launch) + nova = self._get_nova(region) + server = nova.servers.get(launch.instance) From 6ec0ae3015eadb60e8833ad63f80f04f0328ae55 Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Mon, 13 May 2013 14:24:01 -0400 Subject: [PATCH 02/10] Continuing reconciler work --- stacktach/reconciler.py | 113 ++++++++---- tests/unit/test_reconciler.py | 321 ++++++++++++++++++++++++++++++++++ 2 files changed, 402 insertions(+), 32 deletions(-) create mode 100644 tests/unit/test_reconciler.py diff --git a/stacktach/reconciler.py b/stacktach/reconciler.py index c9e18e4..9446ea8 100644 --- a/stacktach/reconciler.py +++ b/stacktach/reconciler.py @@ -1,40 +1,47 @@ +# Copyright (c) 2013 - Rackspace Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import json + +from novaclient.exceptions import NotFound from novaclient.v1_1 import client from stacktach import models +from stacktach import utils -reconciler_config = { - 'nova':{ - 'DFW':{ - 'username': 'm0lt3n', - 'project_id': '724740', - 'api_key': '', - 'auth_url': 'https://identity.api.rackspacecloud.com/v2.0', - 'auth_system': 'rackspace', - }, - 'ORD':{ - 'username': 'm0lt3n', - 'project_id': '724740', - 'api_key': '', - 'auth_url': 'https://identity.api.rackspacecloud.com/v2.0', - 'auth_system': 'rackspace', - }, - - }, - 'region_mapping_loc': '/etc/stacktach/region_mapping.json' -} - -region_mapping = { - 'x': 'DFW' -} +TERMINATED_AT_KEY = 'OS-INST-USG:terminated_at' class Reconciler(object): - def __init__(self, config): - self.config = reconciler_config - self.region_mapping = region_mapping + def __init__(self, config, region_mapping=None): + self.config = config + self.region_mapping = (region_mapping or + Reconciler._load_region_mapping(config)) self.nova_clients = {} + @classmethod + def _load_region_mapping(cls, config): + with open(config['region_mapping_loc']) as f: + return json.load(f) + def _get_nova(self, region): if region in self.nova_clients: return self.nova_clients[region] @@ -56,12 +63,54 @@ class Reconciler(object): if raws.count() == 0: return False raw = raws[0] - return self.region_mapping[str(raw.deployment.name)] + deployment_name = str(raw.deployment.name) + if deployment_name in self.region_mapping: + return self.region_mapping[deployment_name] + else: + return False + + def _reconcile_from_api(self, launch, server): + terminated_at = server._info[TERMINATED_AT_KEY] + terminated_at = utils.str_time_to_unix(terminated_at) + values = { + 'instance': server.id, + 'launched_at': launch.launched_at, + 'deleted_at': terminated_at, + 'instance_type_id': launch.instance_type_id, + 'source': 'reconciler:nova_api', + } + models.InstanceReconcile(**values).save() + + def _reconcile_from_api_not_found(self, launch): + values = { + 'instance': launch.instance, + 'launched_at': launch.launched_at, + 'deleted_at': 1, + 'instance_type_id': launch.instance_type_id, + 'source': 'reconciler:nova_api:not_found', + } + models.InstanceReconcile(**values).save() def missing_exists_for_instance(self, launched_id, - period_beginning, - period_ending): - launch = models.InstanceUsage.objects.get(id=launched_id) + period_beginning): + reconciled = False + launch = models.InstanceUsage.objects.get(launched_id) region = self._region_for_launch(launch) nova = self._get_nova(region) - server = nova.servers.get(launch.instance) + try: + server = nova.servers.get(launch.instance) + if TERMINATED_AT_KEY in server._info: + # Check to see if instance has been deleted + terminated_at = server._info[TERMINATED_AT_KEY] + terminated_at = utils.str_time_to_unix(terminated_at) + + if terminated_at < period_beginning: + # Check to see if instance was deleted before period. + # If so, we shouldn't expect an exists. + self._reconcile_from_api(launch, server) + reconciled = True + except NotFound: + self._reconcile_from_api_not_found(launch) + reconciled = True + + return reconciled diff --git a/tests/unit/test_reconciler.py b/tests/unit/test_reconciler.py new file mode 100644 index 0000000..cf65427 --- /dev/null +++ b/tests/unit/test_reconciler.py @@ -0,0 +1,321 @@ +# Copyright (c) 2013 - Rackspace Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import datetime +import unittest + +import mox +from novaclient.exceptions import NotFound +from novaclient.v1_1 import client as nova_client + +from stacktach import models +from stacktach import reconciler +import utils +from utils import INSTANCE_ID_1 +from utils import REQUEST_ID_1 + + +config = { + 'nova': { + 'RegionOne': { + 'username': 'demo', + 'project_id': '111111', + 'api_key': 'some_key', + 'auth_url': 'https://identity.example.com/v2.0', + 'auth_system': 'keystone', + }, + 'RegionTwo': { + 'username': 'demo', + 'project_id': '111111', + 'api_key': 'some_key', + 'auth_url': 'https://identity.example.com/v2.0', + 'auth_system': 'keystone', + }, + + }, + 'region_mapping_loc': '/etc/stacktach/region_mapping.json', + 'flavor_mapping_loc': '/etc/stacktach/flavor_mapping.json', +} + +region_mapping = { + 'RegionOne.prod.cell1': 'RegionOne', + 'RegionTwo.prod.cell1': 'RegionTwo', +} + + +class ReconcilerTestCase(unittest.TestCase): + def setUp(self): + self.reconciler = reconciler.Reconciler(config, + region_mapping=region_mapping) + self.mox = mox.Mox() + self.mox.StubOutWithMock(models, 'RawData', use_mock_anything=True) + models.RawData.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'Deployment', use_mock_anything=True) + models.Deployment.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'Lifecycle', use_mock_anything=True) + models.Lifecycle.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'Timing', use_mock_anything=True) + models.Timing.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'RequestTracker', + use_mock_anything=True) + models.RequestTracker.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'InstanceUsage', + use_mock_anything=True) + models.InstanceUsage.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'InstanceReconcile', + use_mock_anything=True) + models.InstanceReconcile.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'InstanceDeletes', + use_mock_anything=True) + models.InstanceDeletes.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'InstanceExists', + use_mock_anything=True) + models.InstanceExists.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'JsonReport', use_mock_anything=True) + models.JsonReport.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(nova_client, 'Client', use_mock_anything=True) + + def tearDown(self): + self.mox.UnsetStubs() + + def _mocked_nova_client(self): + nova = self.mox.CreateMockAnything() + nova.servers = self.mox.CreateMockAnything() + return nova + + def test_region_for_launch(self): + launch = self.mox.CreateMockAnything() + launch.request_id = REQUEST_ID_1 + result = self.mox.CreateMockAnything() + models.RawData.objects.filter(request_id=REQUEST_ID_1)\ + .AndReturn(result) + result.count().AndReturn(1) + raw = self.mox.CreateMockAnything() + raw.deployment = self.mox.CreateMockAnything() + raw.deployment.name = 'RegionOne.prod.cell1' + result[0].AndReturn(raw) + self.mox.ReplayAll() + region = self.reconciler._region_for_launch(launch) + self.assertEqual('RegionOne', region) + self.mox.VerifyAll() + + def test_region_for_launch_no_mapping(self): + launch = self.mox.CreateMockAnything() + launch.request_id = REQUEST_ID_1 + result = self.mox.CreateMockAnything() + models.RawData.objects.filter(request_id=REQUEST_ID_1)\ + .AndReturn(result) + result.count().AndReturn(1) + raw = self.mox.CreateMockAnything() + raw.deployment = self.mox.CreateMockAnything() + raw.deployment.name = 'RegionOne.prod.cell2' + result[0].AndReturn(raw) + self.mox.ReplayAll() + region = self.reconciler._region_for_launch(launch) + self.assertFalse(region) + self.mox.VerifyAll() + + def test_region_for_launch_no_raws(self): + launch = self.mox.CreateMockAnything() + launch.request_id = REQUEST_ID_1 + result = self.mox.CreateMockAnything() + models.RawData.objects.filter(request_id=REQUEST_ID_1)\ + .AndReturn(result) + result.count().AndReturn(0) + self.mox.ReplayAll() + region = self.reconciler._region_for_launch(launch) + self.assertFalse(region) + self.mox.VerifyAll() + + def test_get_nova(self): + expected_client = self._mocked_nova_client + nova_client.Client('demo', 'some_key', '111111', + auth_url='https://identity.example.com/v2.0', + auth_system='keystone').AndReturn(expected_client) + self.mox.ReplayAll() + client = self.reconciler._get_nova('RegionOne') + self.assertEqual(expected_client, client) + self.mox.VerifyAll() + + def test_get_nova_already_created(self): + expected_client = self.mox.CreateMockAnything() + nova_client.Client('demo', 'some_key', '111111', + auth_url='https://identity.example.com/v2.0', + auth_system='keystone').AndReturn(expected_client) + self.mox.ReplayAll() + self.reconciler._get_nova('RegionOne') + client = self.reconciler._get_nova('RegionOne') + self.assertEqual(expected_client, client) + self.mox.VerifyAll() + + def test_reconcile_from_api(self): + deleted_at = datetime.datetime.utcnow() + launched_at = deleted_at - datetime.timedelta(hours=4) + launch = self.mox.CreateMockAnything() + launch.instance = INSTANCE_ID_1 + launch.launched_at = utils.decimal_utc(launched_at) + launch.instance_type_id = 1 + server = self.mox.CreateMockAnything() + server.id = INSTANCE_ID_1 + server._info = { + 'OS-INST-USG:terminated_at': str(deleted_at), + } + values = { + 'instance': INSTANCE_ID_1, + 'instance_type_id': 1, + 'launched_at': utils.decimal_utc(launched_at), + 'deleted_at': utils.decimal_utc(deleted_at), + 'source': 'reconciler:nova_api' + } + result = self.mox.CreateMockAnything() + models.InstanceReconcile(**values).AndReturn(result) + result.save() + self.mox.ReplayAll() + self.reconciler._reconcile_from_api(launch, server) + self.mox.VerifyAll() + + def test_reconcile_from_api_not_found(self): + deleted_at = datetime.datetime.utcnow() + launched_at = deleted_at - datetime.timedelta(hours=4) + launch = self.mox.CreateMockAnything() + launch.instance = INSTANCE_ID_1 + launch.launched_at = utils.decimal_utc(launched_at) + launch.instance_type_id = 1 + values = { + 'instance': INSTANCE_ID_1, + 'instance_type_id': 1, + 'launched_at': utils.decimal_utc(launched_at), + 'deleted_at': 1, + 'source': 'reconciler:nova_api:not_found' + } + result = self.mox.CreateMockAnything() + models.InstanceReconcile(**values).AndReturn(result) + result.save() + self.mox.ReplayAll() + self.reconciler._reconcile_from_api_not_found(launch) + self.mox.VerifyAll() + + def test_missing_exists_for_instance(self): + now = datetime.datetime.utcnow() + deleted_at_dt = now - datetime.timedelta(days=2) + beginning_dt = now - datetime.timedelta(days=1) + beginning_dec = utils.decimal_utc(beginning_dt) + + launch = self.mox.CreateMockAnything() + launch.instance = INSTANCE_ID_1 + models.InstanceUsage.objects.get(1).AndReturn(launch) + self.mox.StubOutWithMock(self.reconciler, '_region_for_launch') + self.reconciler._region_for_launch(launch).AndReturn('RegionOne') + + self.mox.StubOutWithMock(self.reconciler, '_get_nova') + nova = self._mocked_nova_client() + self.reconciler._get_nova('RegionOne').AndReturn(nova) + server = self.mox.CreateMockAnything() + server._info = { + 'OS-INST-USG:terminated_at': str(deleted_at_dt), + } + nova.servers.get(INSTANCE_ID_1).AndReturn(server) + + self.mox.StubOutWithMock(self.reconciler, '_reconcile_from_api') + self.reconciler._reconcile_from_api(launch, server) + + self.mox.ReplayAll() + result = self.reconciler.missing_exists_for_instance(1, beginning_dec) + self.assertTrue(result) + self.mox.VerifyAll() + + def test_missing_exists_for_instance_deleted_too_soon(self): + now = datetime.datetime.utcnow() + deleted_at_dt = now - datetime.timedelta(hours=4) + beginning_dt = now - datetime.timedelta(days=1) + beginning_dec = utils.decimal_utc(beginning_dt) + + launch = self.mox.CreateMockAnything() + launch.instance = INSTANCE_ID_1 + models.InstanceUsage.objects.get(1).AndReturn(launch) + self.mox.StubOutWithMock(self.reconciler, '_region_for_launch') + self.reconciler._region_for_launch(launch).AndReturn('RegionOne') + + self.mox.StubOutWithMock(self.reconciler, '_get_nova') + nova = self._mocked_nova_client() + self.reconciler._get_nova('RegionOne').AndReturn(nova) + server = self.mox.CreateMockAnything() + server._info = { + 'OS-INST-USG:terminated_at': str(deleted_at_dt), + } + nova.servers.get(INSTANCE_ID_1).AndReturn(server) + + self.mox.StubOutWithMock(self.reconciler, '_reconcile_from_api') + + self.mox.ReplayAll() + result = self.reconciler.missing_exists_for_instance(1, beginning_dec) + self.assertFalse(result) + self.mox.VerifyAll() + + def test_missing_exists_for_instance_not_deleted(self): + now = datetime.datetime.utcnow() + beginning_dt = now - datetime.timedelta(days=1) + beginning_dec = utils.decimal_utc(beginning_dt) + + launch = self.mox.CreateMockAnything() + launch.instance = INSTANCE_ID_1 + models.InstanceUsage.objects.get(1).AndReturn(launch) + self.mox.StubOutWithMock(self.reconciler, '_region_for_launch') + self.reconciler._region_for_launch(launch).AndReturn('RegionOne') + + self.mox.StubOutWithMock(self.reconciler, '_get_nova') + nova = self._mocked_nova_client() + self.reconciler._get_nova('RegionOne').AndReturn(nova) + server = self.mox.CreateMockAnything() + server._info = {} + nova.servers.get(INSTANCE_ID_1).AndReturn(server) + + self.mox.StubOutWithMock(self.reconciler, '_reconcile_from_api') + + self.mox.ReplayAll() + result = self.reconciler.missing_exists_for_instance(1, beginning_dec) + self.assertFalse(result) + self.mox.VerifyAll() + + def test_missing_exists_for_instance_not_found(self): + now = datetime.datetime.utcnow() + beginning_dt = now - datetime.timedelta(days=1) + beginning_dec = utils.decimal_utc(beginning_dt) + + launch = self.mox.CreateMockAnything() + launch.instance = INSTANCE_ID_1 + models.InstanceUsage.objects.get(1).AndReturn(launch) + self.mox.StubOutWithMock(self.reconciler, '_region_for_launch') + self.reconciler._region_for_launch(launch).AndReturn('RegionOne') + + self.mox.StubOutWithMock(self.reconciler, '_get_nova') + nova = self._mocked_nova_client() + self.reconciler._get_nova('RegionOne').AndReturn(nova) + + nova.servers.get(INSTANCE_ID_1).AndRaise(NotFound(404)) + + self.mox.StubOutWithMock(self.reconciler, + '_reconcile_from_api_not_found') + self.reconciler._reconcile_from_api_not_found(launch) + + self.mox.ReplayAll() + result = self.reconciler.missing_exists_for_instance(1, beginning_dec) + self.assertTrue(result) + self.mox.VerifyAll() From cb65528d95852422f99388eed43beda387df28ee Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Mon, 13 May 2013 17:39:45 -0400 Subject: [PATCH 03/10] Adding reconciler to auditor --- etc/sample_reconciler_config.json | 12 +++++ etc/sample_region_mapping.json | 6 +++ reports/nova_usage_audit.py | 76 +++++++++++++++++++++++-------- stacktach/reconciler.py | 5 +- tests/unit/test_reconciler.py | 36 +++++++++++++-- 5 files changed, 108 insertions(+), 27 deletions(-) create mode 100644 etc/sample_reconciler_config.json create mode 100644 etc/sample_region_mapping.json diff --git a/etc/sample_reconciler_config.json b/etc/sample_reconciler_config.json new file mode 100644 index 0000000..90e5a24 --- /dev/null +++ b/etc/sample_reconciler_config.json @@ -0,0 +1,12 @@ +{ + "nova": { + "RegionOne": { + "username": "admin", + "project_id": "admin", + "api_key": "some_key", + "auth_url": "http://identity.example.com:5000/v2.0", + "auth_system": "keystone" + } + }, + "region_mapping_loc": "/etc/stacktach/region_mapping.json" +} diff --git a/etc/sample_region_mapping.json b/etc/sample_region_mapping.json new file mode 100644 index 0000000..675c817 --- /dev/null +++ b/etc/sample_region_mapping.json @@ -0,0 +1,6 @@ +{ + "RegionOne.dev.global": "RegionOne", + "RegionOne.dev.cell1": "RegionOne", + "RegionTwo.dev.global": "RegionTwo", + "RegionTwo.dev.cell1": "RegionTwo" +} diff --git a/reports/nova_usage_audit.py b/reports/nova_usage_audit.py index 6a7eff4..a4e54f3 100644 --- a/reports/nova_usage_audit.py +++ b/reports/nova_usage_audit.py @@ -30,12 +30,21 @@ from django.db.models import F from stacktach import datetime_to_decimal as dt from stacktach import models +from stacktach.reconciler import Reconciler -OLD_LAUNCHES_QUERY = "select * from stacktach_instanceusage " \ - "where launched_at is not null and " \ - "launched_at < %s and instance not in " \ - "(select distinct(instance) " \ - "from stacktach_instancedeletes where deleted_at < %s)" +OLD_LAUNCHES_QUERY = """ +select * from stacktach_instanceusage where + launched_at is not null and + launched_at < %s and + instance not in + (select distinct(instance) + from stacktach_instancedeletes where + deleted_at < %s union + select distinct(instance) + from stacktach_instancereconcile where + deleted_at < %s);""" + +reconciler = None def _get_new_launches(beginning, ending): @@ -63,25 +72,34 @@ def _get_exists(beginning, ending): return models.InstanceExists.objects.filter(**filters) -def _audit_launches_to_exists(launches, exists): +def _audit_launches_to_exists(launches, exists, beginning): fails = [] for (instance, launches) in launches.items(): if instance in exists: - for launch1 in launches: + for expected in launches: found = False - for launch2 in exists[instance]: - if int(launch1['launched_at']) == int(launch2['launched_at']): + for actual in exists[instance]: + if int(expected['launched_at']) == \ + int(actual['launched_at']): # HACK (apmelton): Truncate the decimal because we may not # have the milliseconds. found = True if not found: + rec = False + if reconciler: + args = (expected['id'], beginning) + rec = reconciler.missing_exists_for_instance(*args) msg = "Couldn't find exists for launch (%s, %s)" - msg = msg % (instance, launch1['launched_at']) - fails.append(['Launch', launch1['id'], msg]) + msg = msg % (instance, expected['launched_at']) + fails.append(['Launch', expected['id'], msg, 'Y' if rec else 'N']) else: + rec = False + if reconciler: + args = (launches[0]['id'], beginning) + rec = reconciler.missing_exists_for_instance(*args) msg = "No exists for instance (%s)" % instance - fails.append(['Launch', '-', msg]) + fails.append(['Launch', '-', msg, 'Y' if rec else 'N']) return fails @@ -175,8 +193,13 @@ def _launch_audit_for_period(beginning, ending): else: launches_dict[instance] = [l, ] - old_launches = models.InstanceUsage.objects.raw(OLD_LAUNCHES_QUERY, - [beginning, beginning]) + # NOTE (apmelton) + # Django's safe substitution doesn't allow dict substitution... + # Thus, we send it 'beginning' three times... + old_launches = models.InstanceUsage.objects\ + .raw(OLD_LAUNCHES_QUERY, + [beginning, beginning, beginning]) + old_launches_dict = {} for launch in old_launches: instance = launch.instance @@ -205,7 +228,8 @@ def _launch_audit_for_period(beginning, ending): exists_dict[instance] = [e, ] launch_to_exists_fails = _audit_launches_to_exists(launches_dict, - exists_dict) + exists_dict, + beginning) return launch_to_exists_fails, new_launches.count(), len(old_launches_dict) @@ -222,11 +246,11 @@ def audit_for_period(beginning, ending): summary = { 'verifier': verify_summary, - 'launch_fails': { - 'total_failures': len(detail), + 'launch_summary': { 'new_launches': new_count, - 'old_launches': old_count - } + 'old_launches': old_count, + 'failures': len(detail) + }, } details = { @@ -276,7 +300,7 @@ def store_results(start, end, summary, details): def make_json_report(summary, details): report = [{'summary': summary}, - ['Object', 'ID', 'Error Description']] + ['Object', 'ID', 'Error Description', 'Reconciled?']] report.extend(details['exist_fails']) report.extend(details['launch_fails']) return json.dumps(report) @@ -302,8 +326,20 @@ if __name__ == '__main__': help="If set to true, report will be stored. " "Otherwise, it will just be printed", type=bool, default=False) + parser.add_argument('--reconcile', + help="Enabled reconciliation", + type=bool, default=False) + parser.add_argument('--reconciler_config', + help="Location of the reconciler config file", + type=str, + default='/etc/stacktach/reconciler-config.json') args = parser.parse_args() + if args.reconcile: + with open(args.reconciler_config) as f: + reconciler_config = json.load(f) + reconciler = Reconciler(reconciler_config) + if args.utcdatetime is not None: time = args.utcdatetime else: diff --git a/stacktach/reconciler.py b/stacktach/reconciler.py index 9446ea8..1c3645f 100644 --- a/stacktach/reconciler.py +++ b/stacktach/reconciler.py @@ -94,12 +94,13 @@ class Reconciler(object): def missing_exists_for_instance(self, launched_id, period_beginning): reconciled = False - launch = models.InstanceUsage.objects.get(launched_id) + launch = models.InstanceUsage.objects.get(id=launched_id) region = self._region_for_launch(launch) nova = self._get_nova(region) try: server = nova.servers.get(launch.instance) - if TERMINATED_AT_KEY in server._info: + if (server.status == 'DELETED' and + TERMINATED_AT_KEY in server._info): # Check to see if instance has been deleted terminated_at = server._info[TERMINATED_AT_KEY] terminated_at = utils.str_time_to_unix(terminated_at) diff --git a/tests/unit/test_reconciler.py b/tests/unit/test_reconciler.py index cf65427..994c5c3 100644 --- a/tests/unit/test_reconciler.py +++ b/tests/unit/test_reconciler.py @@ -51,7 +51,6 @@ config = { }, 'region_mapping_loc': '/etc/stacktach/region_mapping.json', - 'flavor_mapping_loc': '/etc/stacktach/flavor_mapping.json', } region_mapping = { @@ -220,7 +219,7 @@ class ReconcilerTestCase(unittest.TestCase): launch = self.mox.CreateMockAnything() launch.instance = INSTANCE_ID_1 - models.InstanceUsage.objects.get(1).AndReturn(launch) + models.InstanceUsage.objects.get(id=1).AndReturn(launch) self.mox.StubOutWithMock(self.reconciler, '_region_for_launch') self.reconciler._region_for_launch(launch).AndReturn('RegionOne') @@ -228,6 +227,7 @@ class ReconcilerTestCase(unittest.TestCase): nova = self._mocked_nova_client() self.reconciler._get_nova('RegionOne').AndReturn(nova) server = self.mox.CreateMockAnything() + server.status = 'DELETED' server._info = { 'OS-INST-USG:terminated_at': str(deleted_at_dt), } @@ -241,6 +241,32 @@ class ReconcilerTestCase(unittest.TestCase): self.assertTrue(result) self.mox.VerifyAll() + def test_missing_exists_for_instance_non_deleted_status(self): + now = datetime.datetime.utcnow() + beginning_dt = now - datetime.timedelta(days=1) + beginning_dec = utils.decimal_utc(beginning_dt) + + launch = self.mox.CreateMockAnything() + launch.instance = INSTANCE_ID_1 + models.InstanceUsage.objects.get(id=1).AndReturn(launch) + self.mox.StubOutWithMock(self.reconciler, '_region_for_launch') + self.reconciler._region_for_launch(launch).AndReturn('RegionOne') + + self.mox.StubOutWithMock(self.reconciler, '_get_nova') + nova = self._mocked_nova_client() + self.reconciler._get_nova('RegionOne').AndReturn(nova) + server = self.mox.CreateMockAnything() + server.status = 'ACTIVE' + server._info = { + 'OS-INST-USG:terminated_at': None, + } + nova.servers.get(INSTANCE_ID_1).AndReturn(server) + + self.mox.ReplayAll() + result = self.reconciler.missing_exists_for_instance(1, beginning_dec) + self.assertFalse(result) + self.mox.VerifyAll() + def test_missing_exists_for_instance_deleted_too_soon(self): now = datetime.datetime.utcnow() deleted_at_dt = now - datetime.timedelta(hours=4) @@ -249,7 +275,7 @@ class ReconcilerTestCase(unittest.TestCase): launch = self.mox.CreateMockAnything() launch.instance = INSTANCE_ID_1 - models.InstanceUsage.objects.get(1).AndReturn(launch) + models.InstanceUsage.objects.get(id=1).AndReturn(launch) self.mox.StubOutWithMock(self.reconciler, '_region_for_launch') self.reconciler._region_for_launch(launch).AndReturn('RegionOne') @@ -276,7 +302,7 @@ class ReconcilerTestCase(unittest.TestCase): launch = self.mox.CreateMockAnything() launch.instance = INSTANCE_ID_1 - models.InstanceUsage.objects.get(1).AndReturn(launch) + models.InstanceUsage.objects.get(id=1).AndReturn(launch) self.mox.StubOutWithMock(self.reconciler, '_region_for_launch') self.reconciler._region_for_launch(launch).AndReturn('RegionOne') @@ -301,7 +327,7 @@ class ReconcilerTestCase(unittest.TestCase): launch = self.mox.CreateMockAnything() launch.instance = INSTANCE_ID_1 - models.InstanceUsage.objects.get(1).AndReturn(launch) + models.InstanceUsage.objects.get(id=1).AndReturn(launch) self.mox.StubOutWithMock(self.reconciler, '_region_for_launch') self.reconciler._region_for_launch(launch).AndReturn('RegionOne') From 9f5468df7a1a37d16112feb2cd95bc39affc09e5 Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Mon, 20 May 2013 12:59:22 -0400 Subject: [PATCH 04/10] Starting reconcile in verifier --- stacktach/models.py | 2 + tests/unit/test_verifier_db.py | 157 +++++++++++++++++++++++++++++++-- verifier/dbverifier.py | 111 +++++++++++++++++++---- 3 files changed, 248 insertions(+), 22 deletions(-) diff --git a/stacktach/models.py b/stacktach/models.py index 6236695..00d233c 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -120,11 +120,13 @@ class InstanceExists(models.Model): PENDING = 'pending' VERIFYING = 'verifying' VERIFIED = 'verified' + RECONCILED = 'reconciled' FAILED = 'failed' STATUS_CHOICES = [ (PENDING, 'Pending Verification'), (VERIFYING, 'Currently Being Verified'), (VERIFIED, 'Passed Verification'), + (RECONCILED, 'Passed Verification After Reconciliation'), (FAILED, 'Failed Verification'), ] instance = models.CharField(max_length=50, null=True, diff --git a/tests/unit/test_verifier_db.py b/tests/unit/test_verifier_db.py index 54f7def..a10e424 100644 --- a/tests/unit/test_verifier_db.py +++ b/tests/unit/test_verifier_db.py @@ -64,6 +64,9 @@ class VerifierTestCase(unittest.TestCase): self.mox.StubOutWithMock(models, 'InstanceDeletes', use_mock_anything=True) models.InstanceDeletes.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'InstanceReconcile', + use_mock_anything=True) + models.InstanceReconcile.objects = self.mox.CreateMockAnything() self.mox.StubOutWithMock(models, 'InstanceExists', use_mock_anything=True) models.InstanceExists.objects = self.mox.CreateMockAnything() @@ -369,7 +372,8 @@ class VerifierTestCase(unittest.TestCase): dbverifier._verify_for_delete(exist) dbverifier._mark_exist_verified(exist) self.mox.ReplayAll() - dbverifier._verify(exist) + result, exists = dbverifier._verify(exist) + self.assertTrue(result) self.mox.VerifyAll() def test_verify_no_launched_at(self): @@ -381,8 +385,12 @@ class VerifierTestCase(unittest.TestCase): self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') dbverifier._mark_exist_failed(exist, reason="Exists without a launched_at") + self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data') + dbverifier._verify_with_reconciled_data(exist, mox.IgnoreArg())\ + .AndRaise(NotFound('InstanceReconcile', {})) self.mox.ReplayAll() - dbverifier._verify(exist) + result, exists = dbverifier._verify(exist) + self.assertFalse(result) self.mox.VerifyAll() def test_verify_launch_fail(self): @@ -394,9 +402,140 @@ class VerifierTestCase(unittest.TestCase): self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') verify_exception = VerificationException('test') dbverifier._verify_for_launch(exist).AndRaise(verify_exception) + self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data') + dbverifier._verify_with_reconciled_data(exist, verify_exception)\ + .AndRaise(NotFound('InstanceReconcile', {})) dbverifier._mark_exist_failed(exist, reason='test') self.mox.ReplayAll() - dbverifier._verify(exist) + result, exists = dbverifier._verify(exist) + self.assertFalse(result) + self.mox.VerifyAll() + + def test_verify_fail_reconcile_success(self): + exist = self.mox.CreateMockAnything() + exist.launched_at = decimal.Decimal('1.1') + self.mox.StubOutWithMock(dbverifier, '_verify_for_launch') + self.mox.StubOutWithMock(dbverifier, '_verify_for_delete') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') + verify_exception = VerificationException('test') + dbverifier._verify_for_launch(exist).AndRaise(verify_exception) + self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data') + dbverifier._verify_with_reconciled_data(exist, verify_exception) + dbverifier._mark_exist_verified(exist) + self.mox.ReplayAll() + result, exists = dbverifier._verify(exist) + self.assertTrue(result) + self.mox.VerifyAll() + + def test_verify_fail_no_reconciled_data_successful_reconcile(self): + exist = self.mox.CreateMockAnything() + exist.launched_at = decimal.Decimal('1.1') + self.mox.StubOutWithMock(dbverifier, '_verify_for_launch') + self.mox.StubOutWithMock(dbverifier, '_verify_for_delete') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') + verify_exception = VerificationException('test') + dbverifier._verify_for_launch(exist).AndRaise(verify_exception) + self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data') + rec_exception = NotFound("InstanceReconcile", {}) + dbverifier._verify_with_reconciled_data(exist, verify_exception)\ + .AndRaise(rec_exception) + self.mox.StubOutWithMock(dbverifier, '_attempt_reconciliation') + dbverifier._attempt_reconciliation(exist, rec_exception)\ + .AndReturn(True) + dbverifier._mark_exist_verified(exist, + reconciled=True, + reason='test') + self.mox.ReplayAll() + result, exists = dbverifier._verify(exist) + self.assertTrue(result) + self.mox.VerifyAll() + + def test_verify_fail_no_reconciled_data_unsuccessful_reconcile(self): + exist = self.mox.CreateMockAnything() + exist.launched_at = decimal.Decimal('1.1') + self.mox.StubOutWithMock(dbverifier, '_verify_for_launch') + self.mox.StubOutWithMock(dbverifier, '_verify_for_delete') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') + verify_exception = VerificationException('test') + dbverifier._verify_for_launch(exist).AndRaise(verify_exception) + self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data') + rec_exception = NotFound("InstanceReconcile", {}) + dbverifier._verify_with_reconciled_data(exist, verify_exception)\ + .AndRaise(rec_exception) + self.mox.StubOutWithMock(dbverifier, '_attempt_reconciliation') + dbverifier._attempt_reconciliation(exist, rec_exception)\ + .AndReturn(False) + dbverifier._mark_exist_failed(exist, reason='test') + self.mox.ReplayAll() + result, exists = dbverifier._verify(exist) + self.assertFalse(result) + self.mox.VerifyAll() + + def test_verify_fail_bad_reconciled_data_successful_reconcile(self): + exist = self.mox.CreateMockAnything() + exist.launched_at = decimal.Decimal('1.1') + self.mox.StubOutWithMock(dbverifier, '_verify_for_launch') + self.mox.StubOutWithMock(dbverifier, '_verify_for_delete') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') + verify_exception = VerificationException('test') + dbverifier._verify_for_launch(exist).AndRaise(verify_exception) + self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data') + rec_exception = VerificationException("test2") + dbverifier._verify_with_reconciled_data(exist, verify_exception)\ + .AndRaise(rec_exception) + self.mox.StubOutWithMock(dbverifier, '_attempt_reconciliation') + dbverifier._attempt_reconciliation(exist, rec_exception)\ + .AndReturn(True) + dbverifier._mark_exist_verified(exist, + reconciled=True, + reason='test2') + self.mox.ReplayAll() + result, exists = dbverifier._verify(exist) + self.assertTrue(result) + self.mox.VerifyAll() + + def test_verify_fail_bad_reconciled_data_unsuccessful_reconcile(self): + exist = self.mox.CreateMockAnything() + exist.launched_at = decimal.Decimal('1.1') + self.mox.StubOutWithMock(dbverifier, '_verify_for_launch') + self.mox.StubOutWithMock(dbverifier, '_verify_for_delete') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') + verify_exception = VerificationException('test') + dbverifier._verify_for_launch(exist).AndRaise(verify_exception) + self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data') + rec_exception = VerificationException("test2") + dbverifier._verify_with_reconciled_data(exist, verify_exception)\ + .AndRaise(rec_exception) + self.mox.StubOutWithMock(dbverifier, '_attempt_reconciliation') + dbverifier._attempt_reconciliation(exist, rec_exception)\ + .AndReturn(False) + dbverifier._mark_exist_failed(exist, reason='test2') + self.mox.ReplayAll() + result, exists = dbverifier._verify(exist) + self.assertFalse(result) + self.mox.VerifyAll() + + def test_verify_fail_with_reconciled_data_exception(self): + exist = self.mox.CreateMockAnything() + exist.launched_at = decimal.Decimal('1.1') + self.mox.StubOutWithMock(dbverifier, '_verify_for_launch') + self.mox.StubOutWithMock(dbverifier, '_verify_for_delete') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') + verify_exception = VerificationException('test') + dbverifier._verify_for_launch(exist).AndRaise(verify_exception) + self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data') + dbverifier._verify_with_reconciled_data(exist, verify_exception)\ + .AndRaise(Exception()) + dbverifier._mark_exist_failed(exist, reason='Exception') + self.mox.ReplayAll() + result, exists = dbverifier._verify(exist) + self.assertFalse(result) self.mox.VerifyAll() def test_verify_delete_fail(self): @@ -409,9 +548,13 @@ class VerifierTestCase(unittest.TestCase): verify_exception = VerificationException('test') dbverifier._verify_for_launch(exist) dbverifier._verify_for_delete(exist).AndRaise(verify_exception) + self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data') + dbverifier._verify_with_reconciled_data(exist, verify_exception)\ + .AndRaise(NotFound('InstanceReconcile', {})) dbverifier._mark_exist_failed(exist, reason='test') self.mox.ReplayAll() - dbverifier._verify(exist) + result, exists = dbverifier._verify(exist) + self.assertFalse(result) self.mox.VerifyAll() def test_verify_exception_during_launch(self): @@ -424,7 +567,8 @@ class VerifierTestCase(unittest.TestCase): dbverifier._verify_for_launch(exist).AndRaise(Exception()) dbverifier._mark_exist_failed(exist, reason='Exception') self.mox.ReplayAll() - dbverifier._verify(exist) + result, exists = dbverifier._verify(exist) + self.assertFalse(result) self.mox.VerifyAll() def test_verify_exception_during_delete(self): @@ -438,7 +582,8 @@ class VerifierTestCase(unittest.TestCase): dbverifier._verify_for_delete(exist).AndRaise(Exception()) dbverifier._mark_exist_failed(exist, reason='Exception') self.mox.ReplayAll() - dbverifier._verify(exist) + result, exists = dbverifier._verify(exist) + self.assertFalse(result) self.mox.VerifyAll() def test_verify_for_range_without_callback(self): diff --git a/verifier/dbverifier.py b/verifier/dbverifier.py index 7a6f1f8..858877b 100644 --- a/verifier/dbverifier.py +++ b/verifier/dbverifier.py @@ -69,6 +69,15 @@ def _find_launch(instance, launched): return models.InstanceUsage.objects.filter(**params) +def _find_reconcile(instance, launched): + start = launched - datetime.timedelta(microseconds=launched.microsecond) + end = start + datetime.timedelta(microseconds=999999) + params = {'instance': instance, + 'launched_at__gte': dt.dt_to_decimal(start), + 'launched_at__lte': dt.dt_to_decimal(end)} + return models.InstanceReconcile.objects.filter(**params) + + def _find_delete(instance, launched, deleted_max=None): start = launched - datetime.timedelta(microseconds=launched.microsecond) end = start + datetime.timedelta(microseconds=999999) @@ -80,8 +89,16 @@ def _find_delete(instance, launched, deleted_max=None): return models.InstanceDeletes.objects.filter(**params) -def _mark_exist_verified(exist): - exist.status = models.InstanceExists.VERIFIED +def _mark_exist_verified(exist, + reconciled=False, + reason=None): + if not reconciled: + exist.status = models.InstanceExists.VERIFIED + else: + exist.status = models.InstanceExists.RECONCILED + if reason is not None: + exist.fail_reason = reason + exist.save() @@ -136,10 +153,11 @@ def _verify_field_mismatch(exists, launch): launch.tenant) -def _verify_for_launch(exist): - if exist.usage: +def _verify_for_launch(exist, launch=None, launch_type="InstanceUsage"): + + if not launch and exist.usage: launch = exist.usage - else: + elif not launch: if models.InstanceUsage.objects\ .filter(instance=exist.instance).count() > 0: launches = _find_launch(exist.instance, @@ -150,23 +168,22 @@ def _verify_for_launch(exist): 'launched_at': exist.launched_at } if count > 1: - raise AmbiguousResults('InstanceUsage', query) + raise AmbiguousResults(launch_type, query) elif count == 0: - raise NotFound('InstanceUsage', query) + raise NotFound(launch_type, query) launch = launches[0] else: - raise NotFound('InstanceUsage', {'instance': exist.instance}) + raise NotFound(launch_type, {'instance': exist.instance}) _verify_field_mismatch(exist, launch) -def _verify_for_delete(exist): +def _verify_for_delete(exist, delete=None, delete_type="InstanceDelete"): - delete = None - if exist.delete: + if not delete and exist.delete: # We know we have a delete and we have it's id delete = exist.delete - else: + elif not delete: if exist.deleted_at: # We received this exists before the delete, go find it deletes = _find_delete(exist.instance, @@ -178,7 +195,7 @@ def _verify_for_delete(exist): 'instance': exist.instance, 'launched_at': exist.launched_at } - raise NotFound('InstanceDelete', query) + raise NotFound(delete_type, query) else: # We don't know if this is supposed to have a delete or not. # Thus, we need to check if we have a delete for this instance. @@ -190,7 +207,7 @@ def _verify_for_delete(exist): deleted_at_max = dt.dt_from_decimal(exist.audit_period_ending) deletes = _find_delete(exist.instance, launched_at, deleted_at_max) if deletes.count() > 0: - reason = 'Found InstanceDeletes for non-delete exist' + reason = 'Found %ss for non-delete exist' % delete_type raise VerificationException(reason) if delete: @@ -205,6 +222,35 @@ def _verify_for_delete(exist): delete.deleted_at) +def _verify_with_reconciled_data(exist, ex): + if not exist.launched_at: + raise VerificationException("Exists without a launched_at") + + query = models.InstanceReconcile.objects.filter(instance=exist.instance) + if query.count() > 0: + recs = _find_reconcile(exist.instance, + dt.dt_from_decimal(exist.launched_at)) + search_query = {'instance': exist.instance, + 'launched_at': exist.launched_at} + count = recs.count() + if count > 1: + raise AmbiguousResults('InstanceReconcile', search_query) + elif count == 0: + raise NotFound('InstanceReconcile', search_query) + reconcile = recs[0] + else: + raise NotFound('InstanceReconcile', {'instance': exist.instance}) + + _verify_for_launch(exist, launch=reconcile, + launch_type="InstanceReconcile") + _verify_for_delete(exist, delete=reconcile, + delete_type="InstanceReconcile") + + +def _attempt_reconciliation(exists, ex): + pass + + def _verify(exist): verified = False try: @@ -216,8 +262,41 @@ def _verify(exist): verified = True _mark_exist_verified(exist) - except VerificationException, e: - _mark_exist_failed(exist, reason=str(e)) + except VerificationException, orig_e: + # Something is wrong with the InstanceUsage record + try: + # Attempt to verify against reconciled data + _verify_with_reconciled_data(exist, orig_e) + verified = True + _mark_exist_verified(exist) + except NotFound, rec_e: + # No reconciled data available, so let's try to reconcile + if _attempt_reconciliation(exist, rec_e): + # We were able to reconcile the data, but we still need + # to record why it originally failed + verified = True + _mark_exist_verified(exist, + reconciled=True, + reason=str(orig_e)) + else: + # Couldn't reconcile the data, just mark it failed + _mark_exist_failed(exist, reason=str(orig_e)) + except VerificationException, rec_e: + # Reconciled data was available, but it's wrong as well + # Let's try to reconcile again + if _attempt_reconciliation(exist, rec_e): + # We were able to reconcile the data, but we still need + # to record why it failed again + verified = True + _mark_exist_verified(exist, + reconciled=True, + reason=str(rec_e)) + else: + # Couldn't reconcile the data, just mark it failed + _mark_exist_failed(exist, reason=str(rec_e)) + except Exception, rec_e: + _mark_exist_failed(exist, reason=rec_e.__class__.__name__) + LOG.exception(rec_e) except Exception, e: _mark_exist_failed(exist, reason=e.__class__.__name__) LOG.exception(e) From 2f8b54be747a49617c6fd07f0039a05a89d15633 Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Tue, 18 Jun 2013 15:52:27 -0400 Subject: [PATCH 05/10] Using JSONBridge in reconciler --- stacktach/models.py | 7 + stacktach/reconciler.py | 117 --------- stacktach/reconciler/__init__.py | 107 ++++++++ stacktach/reconciler/exceptions.py | 3 + stacktach/reconciler/nova.py | 54 ++++ stacktach/reconciler/utils.py | 9 + tests/unit/test_reconciler.py | 381 ++++++++++++----------------- 7 files changed, 337 insertions(+), 341 deletions(-) delete mode 100644 stacktach/reconciler.py create mode 100644 stacktach/reconciler/__init__.py create mode 100644 stacktach/reconciler/exceptions.py create mode 100644 stacktach/reconciler/nova.py create mode 100644 stacktach/reconciler/utils.py diff --git a/stacktach/models.py b/stacktach/models.py index 00d233c..906b170 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -89,6 +89,13 @@ class InstanceUsage(models.Model): tenant = models.CharField(max_length=50, null=True, blank=True, db_index=True) + def deployment(self): + raws = RawData.objects.filter(request_id=self.request_id) + if raws.count() == 0: + return False + raw = raws[0] + return raw.deployment + class InstanceDeletes(models.Model): instance = models.CharField(max_length=50, null=True, blank=True, db_index=True) diff --git a/stacktach/reconciler.py b/stacktach/reconciler.py deleted file mode 100644 index 1c3645f..0000000 --- a/stacktach/reconciler.py +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright (c) 2013 - Rackspace Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import json - -from novaclient.exceptions import NotFound -from novaclient.v1_1 import client - -from stacktach import models -from stacktach import utils - -TERMINATED_AT_KEY = 'OS-INST-USG:terminated_at' - - -class Reconciler(object): - - def __init__(self, config, region_mapping=None): - self.config = config - self.region_mapping = (region_mapping or - Reconciler._load_region_mapping(config)) - self.nova_clients = {} - - @classmethod - def _load_region_mapping(cls, config): - with open(config['region_mapping_loc']) as f: - return json.load(f) - - def _get_nova(self, region): - if region in self.nova_clients: - return self.nova_clients[region] - - region_cfg = self.config['nova'][region] - region_auth_system = region_cfg.get('auth_system', 'keystone') - - nova = client.Client(region_cfg['username'], region_cfg['api_key'], - region_cfg['project_id'], - auth_url=region_cfg['auth_url'], - auth_system=region_auth_system) - - self.nova_clients[region] = nova - return nova - - def _region_for_launch(self, launch): - request = launch.request_id - raws = models.RawData.objects.filter(request_id=request) - if raws.count() == 0: - return False - raw = raws[0] - deployment_name = str(raw.deployment.name) - if deployment_name in self.region_mapping: - return self.region_mapping[deployment_name] - else: - return False - - def _reconcile_from_api(self, launch, server): - terminated_at = server._info[TERMINATED_AT_KEY] - terminated_at = utils.str_time_to_unix(terminated_at) - values = { - 'instance': server.id, - 'launched_at': launch.launched_at, - 'deleted_at': terminated_at, - 'instance_type_id': launch.instance_type_id, - 'source': 'reconciler:nova_api', - } - models.InstanceReconcile(**values).save() - - def _reconcile_from_api_not_found(self, launch): - values = { - 'instance': launch.instance, - 'launched_at': launch.launched_at, - 'deleted_at': 1, - 'instance_type_id': launch.instance_type_id, - 'source': 'reconciler:nova_api:not_found', - } - models.InstanceReconcile(**values).save() - - def missing_exists_for_instance(self, launched_id, - period_beginning): - reconciled = False - launch = models.InstanceUsage.objects.get(id=launched_id) - region = self._region_for_launch(launch) - nova = self._get_nova(region) - try: - server = nova.servers.get(launch.instance) - if (server.status == 'DELETED' and - TERMINATED_AT_KEY in server._info): - # Check to see if instance has been deleted - terminated_at = server._info[TERMINATED_AT_KEY] - terminated_at = utils.str_time_to_unix(terminated_at) - - if terminated_at < period_beginning: - # Check to see if instance was deleted before period. - # If so, we shouldn't expect an exists. - self._reconcile_from_api(launch, server) - reconciled = True - except NotFound: - self._reconcile_from_api_not_found(launch) - reconciled = True - - return reconciled diff --git a/stacktach/reconciler/__init__.py b/stacktach/reconciler/__init__.py new file mode 100644 index 0000000..24852ad --- /dev/null +++ b/stacktach/reconciler/__init__.py @@ -0,0 +1,107 @@ +# Copyright (c) 2013 - Rackspace Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import json + +from stacktach import models +from stacktach.reconciler import exceptions +from stacktach.reconciler import nova + +DEFAULT_CLIENT = nova.JSONBridgeClient + +CONFIG = { + 'client_class': 'JSONBridgeClient', + 'client': { + 'url': 'http://stack.dev.ramielrowe.com:8080/query/', + 'username': '', + 'password': '', + 'databases': { + 'RegionOne': 'nova', + } + }, + 'region_mapping_loc': '/etc/stacktach/region_mapping.json' +} + + +class Reconciler(object): + + def __init__(self, config, client=None, region_mapping=None): + self.config = config + self.client = (client or Reconciler._load_client(config)) + self.region_mapping = (region_mapping or + Reconciler._load_region_mapping(config)) + + @classmethod + def load_client(cls, config): + client_class = config.get('client_class') + if client_class == 'JSONBridgeClient': + return nova.JSONBridgeClient(config['client']) + else: + return DEFAULT_CLIENT(config['client']) + + @classmethod + def load_region_mapping(cls, config): + with open(config['region_mapping_loc']) as f: + return json.load(f) + + def _region_for_launch(self, launch): + deployment = launch.deployment() + if deployment: + deployment_name = str(deployment.name) + if deployment_name in self.region_mapping: + return self.region_mapping[deployment_name] + else: + return False + else: + return False + + def _reconcile_instance(self, launch, src, + launched_at=None, deleted_at=None, + instance_type_id=None): + values = { + 'instance': launch.instance, + 'launched_at': (launched_at or launch.launched_at), + 'deleted_at': deleted_at, + 'instance_type_id': (instance_type_id or launch.instance_type_id), + 'source': 'reconciler:%s' % src, + } + models.InstanceReconcile(**values).save() + + def missing_exists_for_instance(self, launched_id, + period_beginning): + reconciled = False + launch = models.InstanceUsage.objects.get(id=launched_id) + region = self._region_for_launch(launch) + try: + instance = self.client.get_instance(region, launch.instance) + if instance['deleted'] and instance['deleted_at'] is not None: + # Check to see if instance has been deleted + deleted_at = instance['deleted_at'] + + if deleted_at < period_beginning: + # Check to see if instance was deleted before period. + # If so, we shouldn't expect an exists. + self._reconcile_instance(launch, self.client.src_str, + deleted_at=instance['deleted_at']) + reconciled = True + except exceptions.NotFound: + reconciled = False + + return reconciled diff --git a/stacktach/reconciler/exceptions.py b/stacktach/reconciler/exceptions.py new file mode 100644 index 0000000..e48f49a --- /dev/null +++ b/stacktach/reconciler/exceptions.py @@ -0,0 +1,3 @@ +class NotFound(Exception): + def __init__(self, message="NotFound"): + self.message = message diff --git a/stacktach/reconciler/nova.py b/stacktach/reconciler/nova.py new file mode 100644 index 0000000..aae40b2 --- /dev/null +++ b/stacktach/reconciler/nova.py @@ -0,0 +1,54 @@ +import json + +import requests + +from stacktach import utils as stackutils +from stacktach.reconciler import exceptions +from stacktach.reconciler.utils import empty_reconciler_instance + + +GET_INSTANCE_QUERY = "SELECT * FROM instances where uuid ='%s';" + + +class JSONBridgeClient(object): + src_str = 'json_bridge:nova_db' + + def __init__(self, config): + self.config = config + + def _url_for_region(self, region): + return self.config['url'] + self.config['regions'][region] + + def _do_query(self, region, query): + data = {'sql': query} + credentials = (self.config['username'], self.config['password']) + return requests.post(self._url_for_region(region), data, + verify=False, auth=credentials).json() + + def _to_reconciler_instance(self, instance): + r_instance = empty_reconciler_instance() + r_instance.update({ + 'id': instance['uuid'], + 'instance_type_id': instance['instance_type_id'], + }) + + if instance['launched_at'] is not None: + launched_at = stackutils.str_time_to_unix(instance['launched_at']) + r_instance['launched_at'] = launched_at + + if instance['terminated_at'] is not None: + deleted_at = stackutils.str_time_to_unix(instance['terminated_at']) + r_instance['deleted_at'] = deleted_at + + if instance['deleted'] != 0: + r_instance['deleted'] = True + + return r_instance + + def get_instance(self, region, uuid): + results = self._do_query(region, GET_INSTANCE_QUERY % uuid)['result'] + if len(results) > 0: + return self._to_reconciler_instance(results[0]) + else: + msg = "Couldn't find instance (%s) using JSON Bridge in region (%s)" + raise exceptions.NotFound(msg % (uuid, region)) \ No newline at end of file diff --git a/stacktach/reconciler/utils.py b/stacktach/reconciler/utils.py new file mode 100644 index 0000000..b835d25 --- /dev/null +++ b/stacktach/reconciler/utils.py @@ -0,0 +1,9 @@ +def empty_reconciler_instance(): + r_instance = { + 'id': None, + 'launched_at': None, + 'deleted': False, + 'deleted_at': None, + 'instance_type_ud': None + } + return r_instance diff --git a/tests/unit/test_reconciler.py b/tests/unit/test_reconciler.py index 994c5c3..7ba5d0d 100644 --- a/tests/unit/test_reconciler.py +++ b/tests/unit/test_reconciler.py @@ -22,14 +22,15 @@ import datetime import unittest import mox -from novaclient.exceptions import NotFound -from novaclient.v1_1 import client as nova_client +import requests from stacktach import models from stacktach import reconciler -import utils -from utils import INSTANCE_ID_1 -from utils import REQUEST_ID_1 +from stacktach import utils as stackutils +from stacktach.reconciler import exceptions +from stacktach.reconciler import nova +from tests.unit import utils +from tests.unit.utils import INSTANCE_ID_1 config = { @@ -61,9 +62,12 @@ region_mapping = { class ReconcilerTestCase(unittest.TestCase): def setUp(self): - self.reconciler = reconciler.Reconciler(config, - region_mapping=region_mapping) self.mox = mox.Mox() + self.client = self.mox.CreateMockAnything() + self.client.src_str = 'mocked_client' + self.reconciler = reconciler.Reconciler(config, + client=self.client, + region_mapping=region_mapping) self.mox.StubOutWithMock(models, 'RawData', use_mock_anything=True) models.RawData.objects = self.mox.CreateMockAnything() self.mox.StubOutWithMock(models, 'Deployment', use_mock_anything=True) @@ -89,27 +93,50 @@ class ReconcilerTestCase(unittest.TestCase): models.InstanceExists.objects = self.mox.CreateMockAnything() self.mox.StubOutWithMock(models, 'JsonReport', use_mock_anything=True) models.JsonReport.objects = self.mox.CreateMockAnything() - self.mox.StubOutWithMock(nova_client, 'Client', use_mock_anything=True) def tearDown(self): self.mox.UnsetStubs() - def _mocked_nova_client(self): - nova = self.mox.CreateMockAnything() - nova.servers = self.mox.CreateMockAnything() - return nova + def _fake_reconciler_instance(self, uuid=INSTANCE_ID_1, launched_at=None, + deleted_at=None, deleted=False, + instance_type_id=1): + return { + 'id': uuid, + 'launched_at': launched_at, + 'deleted_at': deleted_at, + 'deleted': deleted, + 'instance_type_id': instance_type_id + } + + def test_load_client_json_bridge(self): + mock_config = self.mox.CreateMockAnything() + config = {'client_class': 'JSONBridgeClient', 'client': mock_config} + nova.JSONBridgeClient(mock_config) + self.mox.ReplayAll() + reconciler.Reconciler.load_client(config) + self.mox.VerifyAll() + + def test_load_client_no_class_loads_default_class(self): + mock_config = self.mox.CreateMockAnything() + config = {'client': mock_config} + nova.JSONBridgeClient(mock_config) + self.mox.ReplayAll() + reconciler.Reconciler.load_client(config) + self.mox.VerifyAll() + + def test_load_client_incorrect_class_loads_default_class(self): + mock_config = self.mox.CreateMockAnything() + config = {'client_class': 'BadConfigValue', 'client': mock_config} + nova.JSONBridgeClient(mock_config) + self.mox.ReplayAll() + reconciler.Reconciler.load_client(config) + self.mox.VerifyAll() def test_region_for_launch(self): launch = self.mox.CreateMockAnything() - launch.request_id = REQUEST_ID_1 - result = self.mox.CreateMockAnything() - models.RawData.objects.filter(request_id=REQUEST_ID_1)\ - .AndReturn(result) - result.count().AndReturn(1) - raw = self.mox.CreateMockAnything() - raw.deployment = self.mox.CreateMockAnything() - raw.deployment.name = 'RegionOne.prod.cell1' - result[0].AndReturn(raw) + deployment = self.mox.CreateMockAnything() + deployment.name = 'RegionOne.prod.cell1' + launch.deployment().AndReturn(deployment) self.mox.ReplayAll() region = self.reconciler._region_for_launch(launch) self.assertEqual('RegionOne', region) @@ -117,15 +144,9 @@ class ReconcilerTestCase(unittest.TestCase): def test_region_for_launch_no_mapping(self): launch = self.mox.CreateMockAnything() - launch.request_id = REQUEST_ID_1 - result = self.mox.CreateMockAnything() - models.RawData.objects.filter(request_id=REQUEST_ID_1)\ - .AndReturn(result) - result.count().AndReturn(1) - raw = self.mox.CreateMockAnything() - raw.deployment = self.mox.CreateMockAnything() - raw.deployment.name = 'RegionOne.prod.cell2' - result[0].AndReturn(raw) + deployment = self.mox.CreateMockAnything() + deployment.name = 'RegionOne.prod.cell2' + launch.deployment().AndReturn(deployment) self.mox.ReplayAll() region = self.reconciler._region_for_launch(launch) self.assertFalse(region) @@ -133,215 +154,127 @@ class ReconcilerTestCase(unittest.TestCase): def test_region_for_launch_no_raws(self): launch = self.mox.CreateMockAnything() - launch.request_id = REQUEST_ID_1 - result = self.mox.CreateMockAnything() - models.RawData.objects.filter(request_id=REQUEST_ID_1)\ - .AndReturn(result) - result.count().AndReturn(0) + launch.deployment() self.mox.ReplayAll() region = self.reconciler._region_for_launch(launch) self.assertFalse(region) self.mox.VerifyAll() - def test_get_nova(self): - expected_client = self._mocked_nova_client - nova_client.Client('demo', 'some_key', '111111', - auth_url='https://identity.example.com/v2.0', - auth_system='keystone').AndReturn(expected_client) - self.mox.ReplayAll() - client = self.reconciler._get_nova('RegionOne') - self.assertEqual(expected_client, client) - self.mox.VerifyAll() - - def test_get_nova_already_created(self): - expected_client = self.mox.CreateMockAnything() - nova_client.Client('demo', 'some_key', '111111', - auth_url='https://identity.example.com/v2.0', - auth_system='keystone').AndReturn(expected_client) - self.mox.ReplayAll() - self.reconciler._get_nova('RegionOne') - client = self.reconciler._get_nova('RegionOne') - self.assertEqual(expected_client, client) - self.mox.VerifyAll() - - def test_reconcile_from_api(self): - deleted_at = datetime.datetime.utcnow() - launched_at = deleted_at - datetime.timedelta(hours=4) - launch = self.mox.CreateMockAnything() - launch.instance = INSTANCE_ID_1 - launch.launched_at = utils.decimal_utc(launched_at) - launch.instance_type_id = 1 - server = self.mox.CreateMockAnything() - server.id = INSTANCE_ID_1 - server._info = { - 'OS-INST-USG:terminated_at': str(deleted_at), - } - values = { - 'instance': INSTANCE_ID_1, - 'instance_type_id': 1, - 'launched_at': utils.decimal_utc(launched_at), - 'deleted_at': utils.decimal_utc(deleted_at), - 'source': 'reconciler:nova_api' - } - result = self.mox.CreateMockAnything() - models.InstanceReconcile(**values).AndReturn(result) - result.save() - self.mox.ReplayAll() - self.reconciler._reconcile_from_api(launch, server) - self.mox.VerifyAll() - - def test_reconcile_from_api_not_found(self): - deleted_at = datetime.datetime.utcnow() - launched_at = deleted_at - datetime.timedelta(hours=4) - launch = self.mox.CreateMockAnything() - launch.instance = INSTANCE_ID_1 - launch.launched_at = utils.decimal_utc(launched_at) - launch.instance_type_id = 1 - values = { - 'instance': INSTANCE_ID_1, - 'instance_type_id': 1, - 'launched_at': utils.decimal_utc(launched_at), - 'deleted_at': 1, - 'source': 'reconciler:nova_api:not_found' - } - result = self.mox.CreateMockAnything() - models.InstanceReconcile(**values).AndReturn(result) - result.save() - self.mox.ReplayAll() - self.reconciler._reconcile_from_api_not_found(launch) - self.mox.VerifyAll() - def test_missing_exists_for_instance(self): - now = datetime.datetime.utcnow() - deleted_at_dt = now - datetime.timedelta(days=2) - beginning_dt = now - datetime.timedelta(days=1) - beginning_dec = utils.decimal_utc(beginning_dt) - + launch_id = 1 + beginning_d = utils.decimal_utc() launch = self.mox.CreateMockAnything() launch.instance = INSTANCE_ID_1 - models.InstanceUsage.objects.get(id=1).AndReturn(launch) - self.mox.StubOutWithMock(self.reconciler, '_region_for_launch') - self.reconciler._region_for_launch(launch).AndReturn('RegionOne') - - self.mox.StubOutWithMock(self.reconciler, '_get_nova') - nova = self._mocked_nova_client() - self.reconciler._get_nova('RegionOne').AndReturn(nova) - server = self.mox.CreateMockAnything() - server.status = 'DELETED' - server._info = { - 'OS-INST-USG:terminated_at': str(deleted_at_dt), + launch.launched_at = beginning_d - (60*60) + launch.instance_type_id = 1 + models.InstanceUsage.objects.get(id=launch_id).AndReturn(launch) + deployment = self.mox.CreateMockAnything() + launch.deployment().AndReturn(deployment) + deployment.name = 'RegionOne.prod.cell1' + deleted_at = beginning_d - (60*30) + rec_inst = self._fake_reconciler_instance(deleted=True, + deleted_at=deleted_at) + self.client.get_instance('RegionOne', INSTANCE_ID_1).AndReturn(rec_inst) + reconcile_vals = { + 'instance': launch.instance, + 'launched_at': launch.launched_at, + 'deleted_at': deleted_at, + 'instance_type_id': launch.instance_type_id, + 'source': 'reconciler:mocked_client' } - nova.servers.get(INSTANCE_ID_1).AndReturn(server) - - self.mox.StubOutWithMock(self.reconciler, '_reconcile_from_api') - self.reconciler._reconcile_from_api(launch, server) - + result = self.mox.CreateMockAnything() + models.InstanceReconcile(**reconcile_vals).AndReturn(result) + result.save() self.mox.ReplayAll() - result = self.reconciler.missing_exists_for_instance(1, beginning_dec) + result = self.reconciler.missing_exists_for_instance(launch_id, + beginning_d) self.assertTrue(result) self.mox.VerifyAll() - def test_missing_exists_for_instance_non_deleted_status(self): - now = datetime.datetime.utcnow() - beginning_dt = now - datetime.timedelta(days=1) - beginning_dec = utils.decimal_utc(beginning_dt) - - launch = self.mox.CreateMockAnything() - launch.instance = INSTANCE_ID_1 - models.InstanceUsage.objects.get(id=1).AndReturn(launch) - self.mox.StubOutWithMock(self.reconciler, '_region_for_launch') - self.reconciler._region_for_launch(launch).AndReturn('RegionOne') - - self.mox.StubOutWithMock(self.reconciler, '_get_nova') - nova = self._mocked_nova_client() - self.reconciler._get_nova('RegionOne').AndReturn(nova) - server = self.mox.CreateMockAnything() - server.status = 'ACTIVE' - server._info = { - 'OS-INST-USG:terminated_at': None, - } - nova.servers.get(INSTANCE_ID_1).AndReturn(server) - - self.mox.ReplayAll() - result = self.reconciler.missing_exists_for_instance(1, beginning_dec) - self.assertFalse(result) - self.mox.VerifyAll() - - def test_missing_exists_for_instance_deleted_too_soon(self): - now = datetime.datetime.utcnow() - deleted_at_dt = now - datetime.timedelta(hours=4) - beginning_dt = now - datetime.timedelta(days=1) - beginning_dec = utils.decimal_utc(beginning_dt) - - launch = self.mox.CreateMockAnything() - launch.instance = INSTANCE_ID_1 - models.InstanceUsage.objects.get(id=1).AndReturn(launch) - self.mox.StubOutWithMock(self.reconciler, '_region_for_launch') - self.reconciler._region_for_launch(launch).AndReturn('RegionOne') - - self.mox.StubOutWithMock(self.reconciler, '_get_nova') - nova = self._mocked_nova_client() - self.reconciler._get_nova('RegionOne').AndReturn(nova) - server = self.mox.CreateMockAnything() - server._info = { - 'OS-INST-USG:terminated_at': str(deleted_at_dt), - } - nova.servers.get(INSTANCE_ID_1).AndReturn(server) - - self.mox.StubOutWithMock(self.reconciler, '_reconcile_from_api') - - self.mox.ReplayAll() - result = self.reconciler.missing_exists_for_instance(1, beginning_dec) - self.assertFalse(result) - self.mox.VerifyAll() - - def test_missing_exists_for_instance_not_deleted(self): - now = datetime.datetime.utcnow() - beginning_dt = now - datetime.timedelta(days=1) - beginning_dec = utils.decimal_utc(beginning_dt) - - launch = self.mox.CreateMockAnything() - launch.instance = INSTANCE_ID_1 - models.InstanceUsage.objects.get(id=1).AndReturn(launch) - self.mox.StubOutWithMock(self.reconciler, '_region_for_launch') - self.reconciler._region_for_launch(launch).AndReturn('RegionOne') - - self.mox.StubOutWithMock(self.reconciler, '_get_nova') - nova = self._mocked_nova_client() - self.reconciler._get_nova('RegionOne').AndReturn(nova) - server = self.mox.CreateMockAnything() - server._info = {} - nova.servers.get(INSTANCE_ID_1).AndReturn(server) - - self.mox.StubOutWithMock(self.reconciler, '_reconcile_from_api') - - self.mox.ReplayAll() - result = self.reconciler.missing_exists_for_instance(1, beginning_dec) - self.assertFalse(result) - self.mox.VerifyAll() - def test_missing_exists_for_instance_not_found(self): - now = datetime.datetime.utcnow() - beginning_dt = now - datetime.timedelta(days=1) - beginning_dec = utils.decimal_utc(beginning_dt) - + launch_id = 1 + beginning_d = utils.decimal_utc() launch = self.mox.CreateMockAnything() launch.instance = INSTANCE_ID_1 - models.InstanceUsage.objects.get(id=1).AndReturn(launch) - self.mox.StubOutWithMock(self.reconciler, '_region_for_launch') - self.reconciler._region_for_launch(launch).AndReturn('RegionOne') - - self.mox.StubOutWithMock(self.reconciler, '_get_nova') - nova = self._mocked_nova_client() - self.reconciler._get_nova('RegionOne').AndReturn(nova) - - nova.servers.get(INSTANCE_ID_1).AndRaise(NotFound(404)) - - self.mox.StubOutWithMock(self.reconciler, - '_reconcile_from_api_not_found') - self.reconciler._reconcile_from_api_not_found(launch) - + launch.launched_at = beginning_d - (60*60) + launch.instance_type_id = 1 + models.InstanceUsage.objects.get(id=launch_id).AndReturn(launch) + deployment = self.mox.CreateMockAnything() + launch.deployment().AndReturn(deployment) + deployment.name = 'RegionOne.prod.cell1' + ex = exceptions.NotFound() + self.client.get_instance('RegionOne', INSTANCE_ID_1).AndRaise(ex) self.mox.ReplayAll() - result = self.reconciler.missing_exists_for_instance(1, beginning_dec) - self.assertTrue(result) + result = self.reconciler.missing_exists_for_instance(launch_id, + beginning_d) + self.assertFalse(result) + self.mox.VerifyAll() + + +json_bridge_config = { + 'url': 'http://json_bridge.example.com/query/', + 'username': 'user', + 'password': 'pass', + 'regions': { + 'RegionOne': 'nova', + } +} + + +class NovaJSONBridgeClientTestCase(unittest.TestCase): + def setUp(self): + self.mox = mox.Mox() + self.client = nova.JSONBridgeClient(json_bridge_config) + self.mox.StubOutWithMock(requests, 'post') + + def tearDown(self): + self.mox.UnsetStubs() + + def mock_for_query(self, database, query, results): + url = json_bridge_config['url'] + database + data = {'sql': query} + auth = (json_bridge_config['username'], json_bridge_config['password']) + result = {'result': results} + response = self.mox.CreateMockAnything() + requests.post(url, data, auth=auth, verify=False)\ + .AndReturn(response) + response.json().AndReturn(result) + + def _fake_instance(self, uuid=INSTANCE_ID_1, launched_at=None, + terminated_at=None, deleted=0, instance_type_id=1): + return { + 'uuid': uuid, + 'launched_at': launched_at, + 'terminated_at': terminated_at, + 'deleted': deleted, + 'instance_type_id': instance_type_id + } + + def test_get_instance(self): + launched_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=5) + launched_at = str(launched_at) + terminated_at = str(datetime.datetime.utcnow()) + results = [self._fake_instance(launched_at=launched_at, + terminated_at=terminated_at, + deleted=True)] + self.mock_for_query('nova', nova.GET_INSTANCE_QUERY % INSTANCE_ID_1, + results) + self.mox.ReplayAll() + instance = self.client.get_instance('RegionOne', INSTANCE_ID_1) + self.assertIsNotNone(instance) + self.assertEqual(instance['id'], INSTANCE_ID_1) + self.assertEqual(instance['instance_type_id'], 1) + launched_at_dec = stackutils.str_time_to_unix(launched_at) + self.assertEqual(instance['launched_at'], launched_at_dec) + terminated_at_dec = stackutils.str_time_to_unix(terminated_at) + self.assertEqual(instance['deleted_at'], terminated_at_dec) + self.assertTrue(instance['deleted']) + self.mox.VerifyAll() + + def test_get_instance_not_found(self): + self.mock_for_query('nova', nova.GET_INSTANCE_QUERY % INSTANCE_ID_1, + []) + self.mox.ReplayAll() + self.assertRaises(exceptions.NotFound, self.client.get_instance, + 'RegionOne', INSTANCE_ID_1) self.mox.VerifyAll() From 0c47be60a697c93ede7fcc4786bc5b220fb722f3 Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Mon, 24 Jun 2013 16:41:46 -0400 Subject: [PATCH 06/10] Finishing up reconciler in auditor --- stacktach/reconciler/__init__.py | 4 ++-- stacktach/reconciler/nova.py | 2 +- tests/unit/test_reconciler.py | 26 ++------------------------ 3 files changed, 5 insertions(+), 27 deletions(-) diff --git a/stacktach/reconciler/__init__.py b/stacktach/reconciler/__init__.py index 24852ad..68b7138 100644 --- a/stacktach/reconciler/__init__.py +++ b/stacktach/reconciler/__init__.py @@ -44,9 +44,9 @@ class Reconciler(object): def __init__(self, config, client=None, region_mapping=None): self.config = config - self.client = (client or Reconciler._load_client(config)) + self.client = (client or Reconciler.load_client(config)) self.region_mapping = (region_mapping or - Reconciler._load_region_mapping(config)) + Reconciler.load_region_mapping(config)) @classmethod def load_client(cls, config): diff --git a/stacktach/reconciler/nova.py b/stacktach/reconciler/nova.py index aae40b2..59e5955 100644 --- a/stacktach/reconciler/nova.py +++ b/stacktach/reconciler/nova.py @@ -17,7 +17,7 @@ class JSONBridgeClient(object): self.config = config def _url_for_region(self, region): - return self.config['url'] + self.config['regions'][region] + return self.config['url'] + self.config['databases'][region] def _do_query(self, region, query): data = {'sql': query} diff --git a/tests/unit/test_reconciler.py b/tests/unit/test_reconciler.py index 7ba5d0d..8ec98fa 100644 --- a/tests/unit/test_reconciler.py +++ b/tests/unit/test_reconciler.py @@ -32,28 +32,6 @@ from stacktach.reconciler import nova from tests.unit import utils from tests.unit.utils import INSTANCE_ID_1 - -config = { - 'nova': { - 'RegionOne': { - 'username': 'demo', - 'project_id': '111111', - 'api_key': 'some_key', - 'auth_url': 'https://identity.example.com/v2.0', - 'auth_system': 'keystone', - }, - 'RegionTwo': { - 'username': 'demo', - 'project_id': '111111', - 'api_key': 'some_key', - 'auth_url': 'https://identity.example.com/v2.0', - 'auth_system': 'keystone', - }, - - }, - 'region_mapping_loc': '/etc/stacktach/region_mapping.json', -} - region_mapping = { 'RegionOne.prod.cell1': 'RegionOne', 'RegionTwo.prod.cell1': 'RegionTwo', @@ -65,7 +43,7 @@ class ReconcilerTestCase(unittest.TestCase): self.mox = mox.Mox() self.client = self.mox.CreateMockAnything() self.client.src_str = 'mocked_client' - self.reconciler = reconciler.Reconciler(config, + self.reconciler = reconciler.Reconciler({}, client=self.client, region_mapping=region_mapping) self.mox.StubOutWithMock(models, 'RawData', use_mock_anything=True) @@ -215,7 +193,7 @@ json_bridge_config = { 'url': 'http://json_bridge.example.com/query/', 'username': 'user', 'password': 'pass', - 'regions': { + 'databases': { 'RegionOne': 'nova', } } From 2ce4a233a750f3f632dcd262fe476e3d250b0fe2 Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Mon, 24 Jun 2013 21:19:51 +0000 Subject: [PATCH 07/10] Adding Reconciler South Migration --- .../0002_auto__add_instancereconcile.py | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 stacktach/migrations/0002_auto__add_instancereconcile.py diff --git a/stacktach/migrations/0002_auto__add_instancereconcile.py b/stacktach/migrations/0002_auto__add_instancereconcile.py new file mode 100644 index 0000000..968e3b0 --- /dev/null +++ b/stacktach/migrations/0002_auto__add_instancereconcile.py @@ -0,0 +1,143 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'InstanceReconcile' + db.create_table(u'stacktach_instancereconcile', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('row_created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)), + ('row_updated', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)), + ('instance', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + ('launched_at', self.gf('django.db.models.fields.DecimalField')(null=True, max_digits=20, decimal_places=6, db_index=True)), + ('deleted_at', self.gf('django.db.models.fields.DecimalField')(null=True, max_digits=20, decimal_places=6, db_index=True)), + ('instance_type_id', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + ('source', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=150, null=True, blank=True)), + )) + db.send_create_signal(u'stacktach', ['InstanceReconcile']) + + + def backwards(self, orm): + # Deleting model 'InstanceReconcile' + db.delete_table(u'stacktach_instancereconcile') + + + models = { + u'stacktach.deployment': { + 'Meta': {'object_name': 'Deployment'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + u'stacktach.instancedeletes': { + 'Meta': {'object_name': 'InstanceDeletes'}, + 'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.RawData']", 'null': 'True'}) + }, + u'stacktach.instanceexists': { + 'Meta': {'object_name': 'InstanceExists'}, + 'audit_period_beginning': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'audit_period_ending': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'delete': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.InstanceDeletes']"}), + 'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'fail_reason': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '300', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'instance_type_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'message_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'raw': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.RawData']"}), + 'send_status': ('django.db.models.fields.IntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True'}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'pending'", 'max_length': '50', 'db_index': 'True'}), + 'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'usage': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.InstanceUsage']"}) + }, + u'stacktach.instancereconcile': { + 'Meta': {'object_name': 'InstanceReconcile'}, + 'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'instance_type_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'row_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'row_updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'source': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '150', 'null': 'True', 'blank': 'True'}) + }, + u'stacktach.instanceusage': { + 'Meta': {'object_name': 'InstanceUsage'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'instance_type_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'request_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}) + }, + u'stacktach.jsonreport': { + 'Meta': {'object_name': 'JsonReport'}, + 'created': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'json': ('django.db.models.fields.TextField', [], {}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}), + 'period_end': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'period_start': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'version': ('django.db.models.fields.IntegerField', [], {'default': '1'}) + }, + u'stacktach.lifecycle': { + 'Meta': {'object_name': 'Lifecycle'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'last_raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.RawData']", 'null': 'True'}), + 'last_state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'last_task_state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}) + }, + u'stacktach.rawdata': { + 'Meta': {'object_name': 'RawData'}, + 'deployment': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Deployment']"}), + 'event': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'host': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'image_type': ('django.db.models.fields.IntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True'}), + 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'json': ('django.db.models.fields.TextField', [], {}), + 'old_state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '20', 'null': 'True', 'blank': 'True'}), + 'old_task': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '30', 'null': 'True', 'blank': 'True'}), + 'publisher': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'request_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'routing_key': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'service': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '20', 'null': 'True', 'blank': 'True'}), + 'task': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '30', 'null': 'True', 'blank': 'True'}), + 'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'when': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}) + }, + u'stacktach.requesttracker': { + 'Meta': {'object_name': 'RequestTracker'}, + 'completed': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'duration': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_timing': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Timing']", 'null': 'True'}), + 'lifecycle': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Lifecycle']"}), + 'request_id': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}), + 'start': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}) + }, + u'stacktach.timing': { + 'Meta': {'object_name': 'Timing'}, + 'diff': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'end_raw': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.RawData']"}), + 'end_when': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'lifecycle': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Lifecycle']"}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}), + 'start_raw': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.RawData']"}), + 'start_when': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6'}) + } + } + + complete_apps = ['stacktach'] \ No newline at end of file From 7f1b9ce313743f75d9b373e8ff91780efca6a343 Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Tue, 25 Jun 2013 10:10:51 -0400 Subject: [PATCH 08/10] Updating sample reconciler config --- etc/sample_reconciler_config.json | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/etc/sample_reconciler_config.json b/etc/sample_reconciler_config.json index 90e5a24..0a19be9 100644 --- a/etc/sample_reconciler_config.json +++ b/etc/sample_reconciler_config.json @@ -1,12 +1,13 @@ { - "nova": { - "RegionOne": { - "username": "admin", - "project_id": "admin", - "api_key": "some_key", - "auth_url": "http://identity.example.com:5000/v2.0", - "auth_system": "keystone" + "client_class": "JSONBridgeClient", + "client": { + "url": "http://jsonbridge.example.com:8080/query/", + "username": "bridgeuser", + "password": "super_secure_password", + "regions": { + "RegionOne": "nova-regionone", + "RegionTwo": "nova-regiontwo" } }, - "region_mapping_loc": "/etc/stacktach/region_mapping.json" -} + "region_mapping_loc": "etc/sample_region_mapping.json" +} \ No newline at end of file From 875be0997561404a33b496990180905751f8efd7 Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Tue, 25 Jun 2013 13:33:59 -0400 Subject: [PATCH 09/10] Removing stub reconciler method in verifier --- tests/unit/test_verifier_db.py | 92 ---------------------------------- verifier/dbverifier.py | 32 ++---------- 2 files changed, 5 insertions(+), 119 deletions(-) diff --git a/tests/unit/test_verifier_db.py b/tests/unit/test_verifier_db.py index a10e424..b635724 100644 --- a/tests/unit/test_verifier_db.py +++ b/tests/unit/test_verifier_db.py @@ -428,98 +428,6 @@ class VerifierTestCase(unittest.TestCase): self.assertTrue(result) self.mox.VerifyAll() - def test_verify_fail_no_reconciled_data_successful_reconcile(self): - exist = self.mox.CreateMockAnything() - exist.launched_at = decimal.Decimal('1.1') - self.mox.StubOutWithMock(dbverifier, '_verify_for_launch') - self.mox.StubOutWithMock(dbverifier, '_verify_for_delete') - self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed') - self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') - verify_exception = VerificationException('test') - dbverifier._verify_for_launch(exist).AndRaise(verify_exception) - self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data') - rec_exception = NotFound("InstanceReconcile", {}) - dbverifier._verify_with_reconciled_data(exist, verify_exception)\ - .AndRaise(rec_exception) - self.mox.StubOutWithMock(dbverifier, '_attempt_reconciliation') - dbverifier._attempt_reconciliation(exist, rec_exception)\ - .AndReturn(True) - dbverifier._mark_exist_verified(exist, - reconciled=True, - reason='test') - self.mox.ReplayAll() - result, exists = dbverifier._verify(exist) - self.assertTrue(result) - self.mox.VerifyAll() - - def test_verify_fail_no_reconciled_data_unsuccessful_reconcile(self): - exist = self.mox.CreateMockAnything() - exist.launched_at = decimal.Decimal('1.1') - self.mox.StubOutWithMock(dbverifier, '_verify_for_launch') - self.mox.StubOutWithMock(dbverifier, '_verify_for_delete') - self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed') - self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') - verify_exception = VerificationException('test') - dbverifier._verify_for_launch(exist).AndRaise(verify_exception) - self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data') - rec_exception = NotFound("InstanceReconcile", {}) - dbverifier._verify_with_reconciled_data(exist, verify_exception)\ - .AndRaise(rec_exception) - self.mox.StubOutWithMock(dbverifier, '_attempt_reconciliation') - dbverifier._attempt_reconciliation(exist, rec_exception)\ - .AndReturn(False) - dbverifier._mark_exist_failed(exist, reason='test') - self.mox.ReplayAll() - result, exists = dbverifier._verify(exist) - self.assertFalse(result) - self.mox.VerifyAll() - - def test_verify_fail_bad_reconciled_data_successful_reconcile(self): - exist = self.mox.CreateMockAnything() - exist.launched_at = decimal.Decimal('1.1') - self.mox.StubOutWithMock(dbverifier, '_verify_for_launch') - self.mox.StubOutWithMock(dbverifier, '_verify_for_delete') - self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed') - self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') - verify_exception = VerificationException('test') - dbverifier._verify_for_launch(exist).AndRaise(verify_exception) - self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data') - rec_exception = VerificationException("test2") - dbverifier._verify_with_reconciled_data(exist, verify_exception)\ - .AndRaise(rec_exception) - self.mox.StubOutWithMock(dbverifier, '_attempt_reconciliation') - dbverifier._attempt_reconciliation(exist, rec_exception)\ - .AndReturn(True) - dbverifier._mark_exist_verified(exist, - reconciled=True, - reason='test2') - self.mox.ReplayAll() - result, exists = dbverifier._verify(exist) - self.assertTrue(result) - self.mox.VerifyAll() - - def test_verify_fail_bad_reconciled_data_unsuccessful_reconcile(self): - exist = self.mox.CreateMockAnything() - exist.launched_at = decimal.Decimal('1.1') - self.mox.StubOutWithMock(dbverifier, '_verify_for_launch') - self.mox.StubOutWithMock(dbverifier, '_verify_for_delete') - self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed') - self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') - verify_exception = VerificationException('test') - dbverifier._verify_for_launch(exist).AndRaise(verify_exception) - self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data') - rec_exception = VerificationException("test2") - dbverifier._verify_with_reconciled_data(exist, verify_exception)\ - .AndRaise(rec_exception) - self.mox.StubOutWithMock(dbverifier, '_attempt_reconciliation') - dbverifier._attempt_reconciliation(exist, rec_exception)\ - .AndReturn(False) - dbverifier._mark_exist_failed(exist, reason='test2') - self.mox.ReplayAll() - result, exists = dbverifier._verify(exist) - self.assertFalse(result) - self.mox.VerifyAll() - def test_verify_fail_with_reconciled_data_exception(self): exist = self.mox.CreateMockAnything() exist.launched_at = decimal.Decimal('1.1') diff --git a/verifier/dbverifier.py b/verifier/dbverifier.py index 858877b..18b028d 100644 --- a/verifier/dbverifier.py +++ b/verifier/dbverifier.py @@ -247,10 +247,6 @@ def _verify_with_reconciled_data(exist, ex): delete_type="InstanceReconcile") -def _attempt_reconciliation(exists, ex): - pass - - def _verify(exist): verified = False try: @@ -270,30 +266,12 @@ def _verify(exist): verified = True _mark_exist_verified(exist) except NotFound, rec_e: - # No reconciled data available, so let's try to reconcile - if _attempt_reconciliation(exist, rec_e): - # We were able to reconcile the data, but we still need - # to record why it originally failed - verified = True - _mark_exist_verified(exist, - reconciled=True, - reason=str(orig_e)) - else: - # Couldn't reconcile the data, just mark it failed - _mark_exist_failed(exist, reason=str(orig_e)) + # No reconciled data, just mark it failed + _mark_exist_failed(exist, reason=str(orig_e)) except VerificationException, rec_e: - # Reconciled data was available, but it's wrong as well - # Let's try to reconcile again - if _attempt_reconciliation(exist, rec_e): - # We were able to reconcile the data, but we still need - # to record why it failed again - verified = True - _mark_exist_verified(exist, - reconciled=True, - reason=str(rec_e)) - else: - # Couldn't reconcile the data, just mark it failed - _mark_exist_failed(exist, reason=str(rec_e)) + # Verification failed against reconciled data, mark it failed + # using the second failure. + _mark_exist_failed(exist, reason=str(rec_e)) except Exception, rec_e: _mark_exist_failed(exist, reason=rec_e.__class__.__name__) LOG.exception(rec_e) From 597eb9f8c5e5ee8aeeb904344c16eca177b71ab3 Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Mon, 1 Jul 2013 17:48:28 +0000 Subject: [PATCH 10/10] Moving instancereconciler migration --- ...cile.py => 0004_create_instancereconcile.py} | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) rename stacktach/migrations/{0002_auto__add_instancereconcile.py => 0004_create_instancereconcile.py} (88%) diff --git a/stacktach/migrations/0002_auto__add_instancereconcile.py b/stacktach/migrations/0004_create_instancereconcile.py similarity index 88% rename from stacktach/migrations/0002_auto__add_instancereconcile.py rename to stacktach/migrations/0004_create_instancereconcile.py index 968e3b0..681c998 100644 --- a/stacktach/migrations/0002_auto__add_instancereconcile.py +++ b/stacktach/migrations/0004_create_instancereconcile.py @@ -53,7 +53,11 @@ class Migration(SchemaMigration): 'instance_type_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), 'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), 'message_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'os_architecture': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'os_distro': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'os_version': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), 'raw': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.RawData']"}), + 'rax_options': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), 'send_status': ('django.db.models.fields.IntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True'}), 'status': ('django.db.models.fields.CharField', [], {'default': "'pending'", 'max_length': '50', 'db_index': 'True'}), 'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), @@ -76,6 +80,10 @@ class Migration(SchemaMigration): 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), 'instance_type_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), 'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'os_architecture': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'os_distro': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'os_version': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'rax_options': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), 'request_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), 'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}) }, @@ -117,6 +125,15 @@ class Migration(SchemaMigration): 'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), 'when': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}) }, + u'stacktach.rawdataimagemeta': { + 'Meta': {'object_name': 'RawDataImageMeta'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'os_architecture': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'os_distro': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'os_version': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.RawData']"}), + 'rax_options': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}) + }, u'stacktach.requesttracker': { 'Meta': {'object_name': 'RequestTracker'}, 'completed': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),