From fe27375b821c7a1f00fb2dd26bf2c7cc10e86cf3 Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Mon, 1 Apr 2013 14:16:35 -0400 Subject: [PATCH] Adding audit_period_beginning/ending to exists table --- migrations/008_delta.sql | 2 + migrations/008_exists_audit_period.py | 58 +++++++++ migrations/008_exists_audit_period.sql | 110 ++++++++++++++++++ .../008_exists_audit_period_indexes.sql | 50 ++++++++ stacktach/models.py | 4 + stacktach/views.py | 4 + tests/unit/test_stacktach.py | 30 +++-- tests/unit/utils.py | 9 +- 8 files changed, 257 insertions(+), 10 deletions(-) create mode 100644 migrations/008_delta.sql create mode 100644 migrations/008_exists_audit_period.py create mode 100644 migrations/008_exists_audit_period.sql create mode 100644 migrations/008_exists_audit_period_indexes.sql diff --git a/migrations/008_delta.sql b/migrations/008_delta.sql new file mode 100644 index 0000000..14fbc21 --- /dev/null +++ b/migrations/008_delta.sql @@ -0,0 +1,2 @@ +ALTER TABLE stacktach_instanceexists ADD `audit_period_beginning` numeric(20, 6); +ALTER TABLE stacktach_instanceexists ADD `audit_period_ending` numeric(20, 6); \ No newline at end of file diff --git a/migrations/008_exists_audit_period.py b/migrations/008_exists_audit_period.py new file mode 100644 index 0000000..dc6be2c --- /dev/null +++ b/migrations/008_exists_audit_period.py @@ -0,0 +1,58 @@ +import datetime +import os +import sys + +try: + import ujson as json +except ImportError: + try: + import simplejson as json + except ImportError: + import json + +POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, os.pardir)) +if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'stacktach')): + sys.path.insert(0, POSSIBLE_TOPDIR) + +from stacktach import models +from stacktach import utils + + +if __name__ != '__main': + sys.exit(1) + + +def print_update(total, completed, errored): + to_go = total - (completed + errored) + print "%s populated, %s to go, %s errored" % (completed, to_go, errored) + + +exists = models.InstanceExists.objects.all() + +count = exists.count() +start = datetime.datetime.utcnow() +print "%s records to populate" + +update_interval = datetime.timedelta(seconds=30) +next_update = start + update_interval + +completed = 0 +errored = 0 +for exist in exists: + try: + notif = json.loads(exist.raw.json) + payload = notif[1]['payload'] + beginning = utils.str_time_to_unix(payload['audit_period_beginning']) + exist.audit_period_beginning = beginning + ending = utils.str_time_to_unix(payload['audit_period_ending']) + exist.audit_period_ending = ending + exist.save() + completed += 1 + except: + print "Error with raw %s" % exist.id + errored += 1 + + if datetime.datetime.utcnow() > next_update: + print_update(count, completed, errored) + next_update = datetime.datetime.utcnow() + update_interval diff --git a/migrations/008_exists_audit_period.sql b/migrations/008_exists_audit_period.sql new file mode 100644 index 0000000..a0b7271 --- /dev/null +++ b/migrations/008_exists_audit_period.sql @@ -0,0 +1,110 @@ +BEGIN; +CREATE TABLE `stacktach_deployment` ( + `id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY, + `name` varchar(50) NOT NULL +) +; +CREATE TABLE `stacktach_rawdata` ( + `id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY, + `deployment_id` integer NOT NULL, + `tenant` varchar(50), + `json` longtext NOT NULL, + `routing_key` varchar(50), + `state` varchar(20), + `old_state` varchar(20), + `old_task` varchar(30), + `task` varchar(30), + `image_type` integer, + `when` numeric(20, 6) NOT NULL, + `publisher` varchar(100), + `event` varchar(50), + `service` varchar(50), + `host` varchar(100), + `instance` varchar(50), + `request_id` varchar(50) +) +; +ALTER TABLE `stacktach_rawdata` ADD CONSTRAINT `deployment_id_refs_id_362370d` FOREIGN KEY (`deployment_id`) REFERENCES `stacktach_deployment` (`id`); +CREATE TABLE `stacktach_lifecycle` ( + `id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY, + `instance` varchar(50), + `last_state` varchar(50), + `last_task_state` varchar(50), + `last_raw_id` integer +) +; +ALTER TABLE `stacktach_lifecycle` ADD CONSTRAINT `last_raw_id_refs_id_d5fb17d3` FOREIGN KEY (`last_raw_id`) REFERENCES `stacktach_rawdata` (`id`); +CREATE TABLE `stacktach_instanceusage` ( + `id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY, + `instance` varchar(50), + `launched_at` numeric(20, 6), + `request_id` varchar(50), + `instance_type_id` varchar(50) +) +; +CREATE TABLE `stacktach_instancedeletes` ( + `id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY, + `instance` varchar(50), + `launched_at` numeric(20, 6), + `deleted_at` numeric(20, 6), + `raw_id` integer +) +; +ALTER TABLE `stacktach_instancedeletes` ADD CONSTRAINT `raw_id_refs_id_58031c62` FOREIGN KEY (`raw_id`) REFERENCES `stacktach_rawdata` (`id`); +CREATE TABLE `stacktach_instanceexists` ( + `id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY, + `instance` varchar(50), + `launched_at` numeric(20, 6), + `deleted_at` numeric(20, 6), + `audit_period_beginning` numeric(20, 6), + `audit_period_ending` numeric(20, 6), + `message_id` varchar(50), + `instance_type_id` varchar(50), + `status` varchar(50) NOT NULL, + `fail_reason` varchar(500), + `raw_id` integer, + `usage_id` integer, + `delete_id` integer, + `send_status` integer +) +; +ALTER TABLE `stacktach_instanceexists` ADD CONSTRAINT `delete_id_refs_id_e02dfe84` FOREIGN KEY (`delete_id`) REFERENCES `stacktach_instancedeletes` (`id`); +ALTER TABLE `stacktach_instanceexists` ADD CONSTRAINT `usage_id_refs_id_c4ecd665` FOREIGN KEY (`usage_id`) REFERENCES `stacktach_instanceusage` (`id`); +ALTER TABLE `stacktach_instanceexists` ADD CONSTRAINT `raw_id_refs_id_65c72953` FOREIGN KEY (`raw_id`) REFERENCES `stacktach_rawdata` (`id`); +CREATE TABLE `stacktach_timing` ( + `id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY, + `name` varchar(50) NOT NULL, + `lifecycle_id` integer NOT NULL, + `start_raw_id` integer, + `end_raw_id` integer, + `start_when` numeric(20, 6), + `end_when` numeric(20, 6), + `diff` numeric(20, 6) +) +; +ALTER TABLE `stacktach_timing` ADD CONSTRAINT `start_raw_id_refs_id_c32dfe04` FOREIGN KEY (`start_raw_id`) REFERENCES `stacktach_rawdata` (`id`); +ALTER TABLE `stacktach_timing` ADD CONSTRAINT `end_raw_id_refs_id_c32dfe04` FOREIGN KEY (`end_raw_id`) REFERENCES `stacktach_rawdata` (`id`); +ALTER TABLE `stacktach_timing` ADD CONSTRAINT `lifecycle_id_refs_id_4255ead8` FOREIGN KEY (`lifecycle_id`) REFERENCES `stacktach_lifecycle` (`id`); +CREATE TABLE `stacktach_requesttracker` ( + `id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY, + `request_id` varchar(50) NOT NULL, + `lifecycle_id` integer NOT NULL, + `last_timing_id` integer, + `start` numeric(20, 6) NOT NULL, + `duration` numeric(20, 6) NOT NULL, + `completed` bool NOT NULL +) +; +ALTER TABLE `stacktach_requesttracker` ADD CONSTRAINT `last_timing_id_refs_id_f0827cca` FOREIGN KEY (`last_timing_id`) REFERENCES `stacktach_timing` (`id`); +ALTER TABLE `stacktach_requesttracker` ADD CONSTRAINT `lifecycle_id_refs_id_e457729` FOREIGN KEY (`lifecycle_id`) REFERENCES `stacktach_lifecycle` (`id`); +CREATE TABLE `stacktach_jsonreport` ( + `id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY, + `period_start` datetime NOT NULL, + `period_end` datetime NOT NULL, + `created` numeric(20, 6) NOT NULL, + `name` varchar(50) NOT NULL, + `version` integer NOT NULL, + `json` longtext NOT NULL +) +; +COMMIT; \ No newline at end of file diff --git a/migrations/008_exists_audit_period_indexes.sql b/migrations/008_exists_audit_period_indexes.sql new file mode 100644 index 0000000..8807871 --- /dev/null +++ b/migrations/008_exists_audit_period_indexes.sql @@ -0,0 +1,50 @@ +BEGIN; +CREATE INDEX `stacktach_rawdata_4ac6801` ON `stacktach_rawdata` (`deployment_id`); +CREATE INDEX `stacktach_rawdata_2207f86d` ON `stacktach_rawdata` (`tenant`); +CREATE INDEX `stacktach_rawdata_2192f43a` ON `stacktach_rawdata` (`routing_key`); +CREATE INDEX `stacktach_rawdata_355bfc27` ON `stacktach_rawdata` (`state`); +CREATE INDEX `stacktach_rawdata_b716e0bb` ON `stacktach_rawdata` (`old_state`); +CREATE INDEX `stacktach_rawdata_8182be12` ON `stacktach_rawdata` (`old_task`); +CREATE INDEX `stacktach_rawdata_1c149b74` ON `stacktach_rawdata` (`task`); +CREATE INDEX `stacktach_rawdata_cfde77eb` ON `stacktach_rawdata` (`image_type`); +CREATE INDEX `stacktach_rawdata_feaed089` ON `stacktach_rawdata` (`when`); +CREATE INDEX `stacktach_rawdata_878a2906` ON `stacktach_rawdata` (`publisher`); +CREATE INDEX `stacktach_rawdata_a90f9116` ON `stacktach_rawdata` (`event`); +CREATE INDEX `stacktach_rawdata_52c5ef6b` ON `stacktach_rawdata` (`service`); +CREATE INDEX `stacktach_rawdata_38dbea87` ON `stacktach_rawdata` (`host`); +CREATE INDEX `stacktach_rawdata_888b756a` ON `stacktach_rawdata` (`instance`); +CREATE INDEX `stacktach_rawdata_792812e8` ON `stacktach_rawdata` (`request_id`); +CREATE INDEX `stacktach_lifecycle_888b756a` ON `stacktach_lifecycle` (`instance`); +CREATE INDEX `stacktach_lifecycle_9b2555fd` ON `stacktach_lifecycle` (`last_state`); +CREATE INDEX `stacktach_lifecycle_67421a0e` ON `stacktach_lifecycle` (`last_task_state`); +CREATE INDEX `stacktach_lifecycle_dcf9e5f3` ON `stacktach_lifecycle` (`last_raw_id`); +CREATE INDEX `stacktach_instanceusage_888b756a` ON `stacktach_instanceusage` (`instance`); +CREATE INDEX `stacktach_instanceusage_792812e8` ON `stacktach_instanceusage` (`request_id`); +CREATE INDEX `stacktach_instanceusage_f321fd7` ON `stacktach_instanceusage` (`instance_type_id`); +CREATE INDEX `stacktach_instancedeletes_888b756a` ON `stacktach_instancedeletes` (`instance`); +CREATE INDEX `stacktach_instancedeletes_365c3a01` ON `stacktach_instancedeletes` (`raw_id`); +CREATE INDEX `stacktach_instanceexists_888b756a` ON `stacktach_instanceexists` (`instance`); +CREATE INDEX `stacktach_instanceexists_38373776` ON `stacktach_instanceexists` (`message_id`); +CREATE INDEX `stacktach_instanceexists_f321fd7` ON `stacktach_instanceexists` (`instance_type_id`); +CREATE INDEX `stacktach_instanceexists_c9ad71dd` ON `stacktach_instanceexists` (`status`); +CREATE INDEX `stacktach_instanceexists_347f3d31` ON `stacktach_instanceexists` (`fail_reason`); +CREATE INDEX `stacktach_instanceexists_365c3a01` ON `stacktach_instanceexists` (`raw_id`); +CREATE INDEX `stacktach_instanceexists_d9ffa990` ON `stacktach_instanceexists` (`usage_id`); +CREATE INDEX `stacktach_instanceexists_cb6f05a7` ON `stacktach_instanceexists` (`delete_id`); +CREATE INDEX `stacktach_instanceexists_b2444339` ON `stacktach_instanceexists` (`send_status`); +CREATE INDEX `stacktach_timing_52094d6e` ON `stacktach_timing` (`name`); +CREATE INDEX `stacktach_timing_9f222e6b` ON `stacktach_timing` (`lifecycle_id`); +CREATE INDEX `stacktach_timing_efab905a` ON `stacktach_timing` (`start_raw_id`); +CREATE INDEX `stacktach_timing_c8bb8daf` ON `stacktach_timing` (`end_raw_id`); +CREATE INDEX `stacktach_timing_4401d15e` ON `stacktach_timing` (`diff`); +CREATE INDEX `stacktach_requesttracker_792812e8` ON `stacktach_requesttracker` (`request_id`); +CREATE INDEX `stacktach_requesttracker_9f222e6b` ON `stacktach_requesttracker` (`lifecycle_id`); +CREATE INDEX `stacktach_requesttracker_ce616a96` ON `stacktach_requesttracker` (`last_timing_id`); +CREATE INDEX `stacktach_requesttracker_29f4f2ea` ON `stacktach_requesttracker` (`start`); +CREATE INDEX `stacktach_requesttracker_8eb45f9b` ON `stacktach_requesttracker` (`duration`); +CREATE INDEX `stacktach_requesttracker_e490d511` ON `stacktach_requesttracker` (`completed`); +CREATE INDEX `stacktach_jsonreport_70ecb89f` ON `stacktach_jsonreport` (`period_start`); +CREATE INDEX `stacktach_jsonreport_6a26a758` ON `stacktach_jsonreport` (`period_end`); +CREATE INDEX `stacktach_jsonreport_3216ff68` ON `stacktach_jsonreport` (`created`); +CREATE INDEX `stacktach_jsonreport_52094d6e` ON `stacktach_jsonreport` (`name`); +COMMIT; \ No newline at end of file diff --git a/stacktach/models.py b/stacktach/models.py index 8d43178..b63f5f2 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -115,6 +115,10 @@ class InstanceExists(models.Model): decimal_places=6) deleted_at = models.DecimalField(null=True, max_digits=20, decimal_places=6) + audit_period_beginning = models.DecimalField(null=True, max_digits=20, + decimal_places=6) + audit_period_ending = models.DecimalField(null=True, max_digits=20, + decimal_places=6) message_id = models.CharField(max_length=50, null=True, blank=True, db_index=True) instance_type_id = models.CharField(max_length=50, diff --git a/stacktach/views.py b/stacktach/views.py index 6f0577d..ab44898 100644 --- a/stacktach/views.py +++ b/stacktach/views.py @@ -286,6 +286,10 @@ def _process_exists(raw, body): values['message_id'] = body['message_id'] values['instance'] = instance_id values['launched_at'] = launched_at + beginning = utils.str_time_to_unix(payload['audit_period_beginning']) + values['audit_period_beginning'] = beginning + ending = utils.str_time_to_unix(payload['audit_period_ending']) + values['audit_period_ending'] = ending values['instance_type_id'] = payload['instance_type_id'] if usage: values['usage'] = usage diff --git a/tests/unit/test_stacktach.py b/tests/unit/test_stacktach.py index 347854d..ca617f9 100644 --- a/tests/unit/test_stacktach.py +++ b/tests/unit/test_stacktach.py @@ -571,11 +571,16 @@ class StacktackUsageParsingTestCase(unittest.TestCase): self.mox.VerifyAll() def test_process_exists(self): - launch_time = datetime.datetime.utcnow()-datetime.timedelta(hours=23) - launch_decimal = utils.decimal_utc(launch_time) current_time = datetime.datetime.utcnow() + launch_time = current_time - datetime.timedelta(hours=23) + launch_decimal = utils.decimal_utc(launch_time) current_decimal = utils.decimal_utc(current_time) - notif = utils.create_nova_notif(launched=str(launch_time)) + audit_beginning = current_time - datetime.timedelta(hours=20) + audit_beginning_decimal = utils.decimal_utc(audit_beginning) + audit_ending_decimal = utils.decimal_utc(current_time) + notif = utils.create_nova_notif(launched=str(launch_time), + audit_period_beginning=str(audit_beginning), + audit_period_ending=str(current_time)) json_str = json.dumps(notif) event = 'compute.instance.exists' raw = utils.create_raw(self.mox, current_decimal, event=event, @@ -592,6 +597,8 @@ class StacktackUsageParsingTestCase(unittest.TestCase): 'message_id': MESSAGE_ID_1, 'instance': INSTANCE_ID_1, 'launched_at': launch_decimal, + 'audit_period_beginning': audit_beginning_decimal, + 'audit_period_ending': audit_ending_decimal, 'instance_type_id': '1', 'usage': usage, 'raw': raw, @@ -604,14 +611,19 @@ class StacktackUsageParsingTestCase(unittest.TestCase): self.mox.VerifyAll() def test_process_exists_with_deleted_at(self): - launch_time = datetime.datetime.utcnow()-datetime.timedelta(hours=23) - launch_decimal = utils.decimal_utc(launch_time) - deleted_time = datetime.datetime.utcnow()-datetime.timedelta(hours=12) - deleted_decimal = utils.decimal_utc(deleted_time) current_time = datetime.datetime.utcnow() + launch_time = current_time - datetime.timedelta(hours=23) + launch_decimal = utils.decimal_utc(launch_time) + deleted_time = current_time - datetime.timedelta(hours=12) + deleted_decimal = utils.decimal_utc(deleted_time) current_decimal = utils.decimal_utc(current_time) + audit_beginning = current_time - datetime.timedelta(hours=20) + audit_beginning_decimal = utils.decimal_utc(audit_beginning) + audit_ending_decimal = utils.decimal_utc(current_time) notif = utils.create_nova_notif(launched=str(launch_time), - deleted=str(deleted_time)) + deleted=str(deleted_time), + audit_period_beginning=str(audit_beginning), + audit_period_ending=str(current_time)) json_str = json.dumps(notif) event = 'compute.instance.exists' raw = utils.create_raw(self.mox, current_decimal, event=event, @@ -630,6 +642,8 @@ class StacktackUsageParsingTestCase(unittest.TestCase): 'instance': INSTANCE_ID_1, 'launched_at': launch_decimal, 'deleted_at': deleted_decimal, + 'audit_period_beginning': audit_beginning_decimal, + 'audit_period_ending': audit_ending_decimal, 'instance_type_id': '1', 'usage': usage, 'delete': delete, diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 43a1da5..5dc9d4c 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -43,8 +43,9 @@ def decimal_utc(t = datetime.datetime.utcnow()): def create_nova_notif(request_id=None, instance=INSTANCE_ID_1, type_id='1', - launched=None, deleted = None, new_type_id=None, - message_id=MESSAGE_ID_1): + launched=None, deleted=None, new_type_id=None, + message_id=MESSAGE_ID_1, audit_period_beginning=None, + audit_period_ending=None): notif = ['', { 'message_id': message_id, 'payload': { @@ -61,6 +62,10 @@ def create_nova_notif(request_id=None, instance=INSTANCE_ID_1, type_id='1', notif[1]['payload']['deleted_at'] = deleted if new_type_id: notif[1]['payload']['new_instance_type_id'] = new_type_id + if audit_period_beginning: + notif[1]['payload']['audit_period_beginning'] = audit_period_beginning + if audit_period_ending: + notif[1]['payload']['audit_period_ending'] = audit_period_ending return notif