Adding audit_period_beginning/ending to exists table

migrations/008_delta.sql

@@ -0,0 +1,2 @@
+ALTER TABLE stacktach_instanceexists ADD `audit_period_beginning` numeric(20, 6);
+ALTER TABLE stacktach_instanceexists ADD `audit_period_ending` numeric(20, 6);

migrations/008_exists_audit_period.py

@@ -0,0 +1,58 @@
+import datetime
+import os
+import sys
+
+try:
+    import ujson as json
+except ImportError:
+    try:
+        import simplejson as json
+    except ImportError:
+        import json
+
+POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
+                                                os.pardir, os.pardir))
+if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'stacktach')):
+    sys.path.insert(0, POSSIBLE_TOPDIR)
+
+from stacktach import models
+from stacktach import utils
+
+
+if __name__ != '__main':
+    sys.exit(1)
+
+
+def print_update(total, completed, errored):
+    to_go = total - (completed + errored)
+    print "%s populated, %s to go, %s errored" % (completed, to_go, errored)
+
+
+exists = models.InstanceExists.objects.all()
+
+count = exists.count()
+start = datetime.datetime.utcnow()
+print "%s records to populate"
+
+update_interval = datetime.timedelta(seconds=30)
+next_update = start + update_interval
+
+completed = 0
+errored = 0
+for exist in exists:
+    try:
+        notif = json.loads(exist.raw.json)
+        payload = notif[1]['payload']
+        beginning = utils.str_time_to_unix(payload['audit_period_beginning'])
+        exist.audit_period_beginning = beginning
+        ending = utils.str_time_to_unix(payload['audit_period_ending'])
+        exist.audit_period_ending = ending
+        exist.save()
+        completed += 1
+    except:
+        print "Error with raw %s" % exist.id
+        errored += 1
+
+    if datetime.datetime.utcnow() > next_update:
+        print_update(count, completed, errored)
+        next_update = datetime.datetime.utcnow() + update_interval

migrations/008_exists_audit_period.sql

@@ -0,0 +1,110 @@
+BEGIN;
+CREATE TABLE `stacktach_deployment` (
+    `id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
+    `name` varchar(50) NOT NULL
+)
+;
+CREATE TABLE `stacktach_rawdata` (
+    `id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
+    `deployment_id` integer NOT NULL,
+    `tenant` varchar(50),
+    `json` longtext NOT NULL,
+    `routing_key` varchar(50),
+    `state` varchar(20),
+    `old_state` varchar(20),
+    `old_task` varchar(30),
+    `task` varchar(30),
+    `image_type` integer,
+    `when` numeric(20, 6) NOT NULL,
+    `publisher` varchar(100),
+    `event` varchar(50),
+    `service` varchar(50),
+    `host` varchar(100),
+    `instance` varchar(50),
+    `request_id` varchar(50)
+)
+;
+ALTER TABLE `stacktach_rawdata` ADD CONSTRAINT `deployment_id_refs_id_362370d` FOREIGN KEY (`deployment_id`) REFERENCES `stacktach_deployment` (`id`);
+CREATE TABLE `stacktach_lifecycle` (
+    `id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
+    `instance` varchar(50),
+    `last_state` varchar(50),
+    `last_task_state` varchar(50),
+    `last_raw_id` integer
+)
+;
+ALTER TABLE `stacktach_lifecycle` ADD CONSTRAINT `last_raw_id_refs_id_d5fb17d3` FOREIGN KEY (`last_raw_id`) REFERENCES `stacktach_rawdata` (`id`);
+CREATE TABLE `stacktach_instanceusage` (
+    `id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
+    `instance` varchar(50),
+    `launched_at` numeric(20, 6),
+    `request_id` varchar(50),
+    `instance_type_id` varchar(50)
+)
+;
+CREATE TABLE `stacktach_instancedeletes` (
+    `id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
+    `instance` varchar(50),
+    `launched_at` numeric(20, 6),
+    `deleted_at` numeric(20, 6),
+    `raw_id` integer
+)
+;
+ALTER TABLE `stacktach_instancedeletes` ADD CONSTRAINT `raw_id_refs_id_58031c62` FOREIGN KEY (`raw_id`) REFERENCES `stacktach_rawdata` (`id`);
+CREATE TABLE `stacktach_instanceexists` (
+    `id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
+    `instance` varchar(50),
+    `launched_at` numeric(20, 6),
+    `deleted_at` numeric(20, 6),
+    `audit_period_beginning` numeric(20, 6),
+    `audit_period_ending` numeric(20, 6),
+    `message_id` varchar(50),
+    `instance_type_id` varchar(50),
+    `status` varchar(50) NOT NULL,
+    `fail_reason` varchar(500),
+    `raw_id` integer,
+    `usage_id` integer,
+    `delete_id` integer,
+    `send_status` integer
+)
+;
+ALTER TABLE `stacktach_instanceexists` ADD CONSTRAINT `delete_id_refs_id_e02dfe84` FOREIGN KEY (`delete_id`) REFERENCES `stacktach_instancedeletes` (`id`);
+ALTER TABLE `stacktach_instanceexists` ADD CONSTRAINT `usage_id_refs_id_c4ecd665` FOREIGN KEY (`usage_id`) REFERENCES `stacktach_instanceusage` (`id`);
+ALTER TABLE `stacktach_instanceexists` ADD CONSTRAINT `raw_id_refs_id_65c72953` FOREIGN KEY (`raw_id`) REFERENCES `stacktach_rawdata` (`id`);
+CREATE TABLE `stacktach_timing` (
+    `id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
+    `name` varchar(50) NOT NULL,
+    `lifecycle_id` integer NOT NULL,
+    `start_raw_id` integer,
+    `end_raw_id` integer,
+    `start_when` numeric(20, 6),
+    `end_when` numeric(20, 6),
+    `diff` numeric(20, 6)
+)
+;
+ALTER TABLE `stacktach_timing` ADD CONSTRAINT `start_raw_id_refs_id_c32dfe04` FOREIGN KEY (`start_raw_id`) REFERENCES `stacktach_rawdata` (`id`);
+ALTER TABLE `stacktach_timing` ADD CONSTRAINT `end_raw_id_refs_id_c32dfe04` FOREIGN KEY (`end_raw_id`) REFERENCES `stacktach_rawdata` (`id`);
+ALTER TABLE `stacktach_timing` ADD CONSTRAINT `lifecycle_id_refs_id_4255ead8` FOREIGN KEY (`lifecycle_id`) REFERENCES `stacktach_lifecycle` (`id`);
+CREATE TABLE `stacktach_requesttracker` (
+    `id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
+    `request_id` varchar(50) NOT NULL,
+    `lifecycle_id` integer NOT NULL,
+    `last_timing_id` integer,
+    `start` numeric(20, 6) NOT NULL,
+    `duration` numeric(20, 6) NOT NULL,
+    `completed` bool NOT NULL
+)
+;
+ALTER TABLE `stacktach_requesttracker` ADD CONSTRAINT `last_timing_id_refs_id_f0827cca` FOREIGN KEY (`last_timing_id`) REFERENCES `stacktach_timing` (`id`);
+ALTER TABLE `stacktach_requesttracker` ADD CONSTRAINT `lifecycle_id_refs_id_e457729` FOREIGN KEY (`lifecycle_id`) REFERENCES `stacktach_lifecycle` (`id`);
+CREATE TABLE `stacktach_jsonreport` (
+    `id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
+    `period_start` datetime NOT NULL,
+    `period_end` datetime NOT NULL,
+    `created` numeric(20, 6) NOT NULL,
+    `name` varchar(50) NOT NULL,
+    `version` integer NOT NULL,
+    `json` longtext NOT NULL
+)
+;
+COMMIT;

migrations/008_exists_audit_period_indexes.sql

@@ -0,0 +1,50 @@
+BEGIN;
+CREATE INDEX `stacktach_rawdata_4ac6801` ON `stacktach_rawdata` (`deployment_id`);
+CREATE INDEX `stacktach_rawdata_2207f86d` ON `stacktach_rawdata` (`tenant`);
+CREATE INDEX `stacktach_rawdata_2192f43a` ON `stacktach_rawdata` (`routing_key`);
+CREATE INDEX `stacktach_rawdata_355bfc27` ON `stacktach_rawdata` (`state`);
+CREATE INDEX `stacktach_rawdata_b716e0bb` ON `stacktach_rawdata` (`old_state`);
+CREATE INDEX `stacktach_rawdata_8182be12` ON `stacktach_rawdata` (`old_task`);
+CREATE INDEX `stacktach_rawdata_1c149b74` ON `stacktach_rawdata` (`task`);
+CREATE INDEX `stacktach_rawdata_cfde77eb` ON `stacktach_rawdata` (`image_type`);
+CREATE INDEX `stacktach_rawdata_feaed089` ON `stacktach_rawdata` (`when`);
+CREATE INDEX `stacktach_rawdata_878a2906` ON `stacktach_rawdata` (`publisher`);
+CREATE INDEX `stacktach_rawdata_a90f9116` ON `stacktach_rawdata` (`event`);
+CREATE INDEX `stacktach_rawdata_52c5ef6b` ON `stacktach_rawdata` (`service`);
+CREATE INDEX `stacktach_rawdata_38dbea87` ON `stacktach_rawdata` (`host`);
+CREATE INDEX `stacktach_rawdata_888b756a` ON `stacktach_rawdata` (`instance`);
+CREATE INDEX `stacktach_rawdata_792812e8` ON `stacktach_rawdata` (`request_id`);
+CREATE INDEX `stacktach_lifecycle_888b756a` ON `stacktach_lifecycle` (`instance`);
+CREATE INDEX `stacktach_lifecycle_9b2555fd` ON `stacktach_lifecycle` (`last_state`);
+CREATE INDEX `stacktach_lifecycle_67421a0e` ON `stacktach_lifecycle` (`last_task_state`);
+CREATE INDEX `stacktach_lifecycle_dcf9e5f3` ON `stacktach_lifecycle` (`last_raw_id`);
+CREATE INDEX `stacktach_instanceusage_888b756a` ON `stacktach_instanceusage` (`instance`);
+CREATE INDEX `stacktach_instanceusage_792812e8` ON `stacktach_instanceusage` (`request_id`);
+CREATE INDEX `stacktach_instanceusage_f321fd7` ON `stacktach_instanceusage` (`instance_type_id`);
+CREATE INDEX `stacktach_instancedeletes_888b756a` ON `stacktach_instancedeletes` (`instance`);
+CREATE INDEX `stacktach_instancedeletes_365c3a01` ON `stacktach_instancedeletes` (`raw_id`);
+CREATE INDEX `stacktach_instanceexists_888b756a` ON `stacktach_instanceexists` (`instance`);
+CREATE INDEX `stacktach_instanceexists_38373776` ON `stacktach_instanceexists` (`message_id`);
+CREATE INDEX `stacktach_instanceexists_f321fd7` ON `stacktach_instanceexists` (`instance_type_id`);
+CREATE INDEX `stacktach_instanceexists_c9ad71dd` ON `stacktach_instanceexists` (`status`);
+CREATE INDEX `stacktach_instanceexists_347f3d31` ON `stacktach_instanceexists` (`fail_reason`);
+CREATE INDEX `stacktach_instanceexists_365c3a01` ON `stacktach_instanceexists` (`raw_id`);
+CREATE INDEX `stacktach_instanceexists_d9ffa990` ON `stacktach_instanceexists` (`usage_id`);
+CREATE INDEX `stacktach_instanceexists_cb6f05a7` ON `stacktach_instanceexists` (`delete_id`);
+CREATE INDEX `stacktach_instanceexists_b2444339` ON `stacktach_instanceexists` (`send_status`);
+CREATE INDEX `stacktach_timing_52094d6e` ON `stacktach_timing` (`name`);
+CREATE INDEX `stacktach_timing_9f222e6b` ON `stacktach_timing` (`lifecycle_id`);
+CREATE INDEX `stacktach_timing_efab905a` ON `stacktach_timing` (`start_raw_id`);
+CREATE INDEX `stacktach_timing_c8bb8daf` ON `stacktach_timing` (`end_raw_id`);
+CREATE INDEX `stacktach_timing_4401d15e` ON `stacktach_timing` (`diff`);
+CREATE INDEX `stacktach_requesttracker_792812e8` ON `stacktach_requesttracker` (`request_id`);
+CREATE INDEX `stacktach_requesttracker_9f222e6b` ON `stacktach_requesttracker` (`lifecycle_id`);
+CREATE INDEX `stacktach_requesttracker_ce616a96` ON `stacktach_requesttracker` (`last_timing_id`);
+CREATE INDEX `stacktach_requesttracker_29f4f2ea` ON `stacktach_requesttracker` (`start`);
+CREATE INDEX `stacktach_requesttracker_8eb45f9b` ON `stacktach_requesttracker` (`duration`);
+CREATE INDEX `stacktach_requesttracker_e490d511` ON `stacktach_requesttracker` (`completed`);
+CREATE INDEX `stacktach_jsonreport_70ecb89f` ON `stacktach_jsonreport` (`period_start`);
+CREATE INDEX `stacktach_jsonreport_6a26a758` ON `stacktach_jsonreport` (`period_end`);
+CREATE INDEX `stacktach_jsonreport_3216ff68` ON `stacktach_jsonreport` (`created`);
+CREATE INDEX `stacktach_jsonreport_52094d6e` ON `stacktach_jsonreport` (`name`);
+COMMIT;

stacktach/models.py

@@ -115,6 +115,10 @@ class InstanceExists(models.Model):
                                       decimal_places=6)
     deleted_at = models.DecimalField(null=True, max_digits=20,
                                      decimal_places=6)
+    audit_period_beginning = models.DecimalField(null=True, max_digits=20,
+                                                 decimal_places=6)
+    audit_period_ending = models.DecimalField(null=True, max_digits=20,
+                                              decimal_places=6)
     message_id = models.CharField(max_length=50, null=True,
                                   blank=True, db_index=True)
     instance_type_id = models.CharField(max_length=50,

stacktach/views.py

@@ -286,6 +286,10 @@ def _process_exists(raw, body):
     values['message_id'] = body['message_id']
     values['instance'] = instance_id
     values['launched_at'] = launched_at
+    beginning = utils.str_time_to_unix(payload['audit_period_beginning'])
+    values['audit_period_beginning'] = beginning
+    ending = utils.str_time_to_unix(payload['audit_period_ending'])
+    values['audit_period_ending'] = ending
     values['instance_type_id'] = payload['instance_type_id']
     if usage:
         values['usage'] = usage

tests/unit/test_stacktach.py

@@ -571,11 +571,16 @@ class StacktackUsageParsingTestCase(unittest.TestCase):
         self.mox.VerifyAll()
 
     def test_process_exists(self):
-        launch_time = datetime.datetime.utcnow()-datetime.timedelta(hours=23)
-        launch_decimal = utils.decimal_utc(launch_time)
         current_time = datetime.datetime.utcnow()
+        launch_time = current_time - datetime.timedelta(hours=23)
+        launch_decimal = utils.decimal_utc(launch_time)
         current_decimal = utils.decimal_utc(current_time)
-        notif = utils.create_nova_notif(launched=str(launch_time))
+        audit_beginning = current_time - datetime.timedelta(hours=20)
+        audit_beginning_decimal = utils.decimal_utc(audit_beginning)
+        audit_ending_decimal = utils.decimal_utc(current_time)
+        notif = utils.create_nova_notif(launched=str(launch_time),
+                                        audit_period_beginning=str(audit_beginning),
+                                        audit_period_ending=str(current_time))
         json_str = json.dumps(notif)
         event = 'compute.instance.exists'
         raw = utils.create_raw(self.mox, current_decimal, event=event,
@@ -592,6 +597,8 @@ class StacktackUsageParsingTestCase(unittest.TestCase):
             'message_id': MESSAGE_ID_1,
             'instance': INSTANCE_ID_1,
             'launched_at': launch_decimal,
+            'audit_period_beginning': audit_beginning_decimal,
+            'audit_period_ending': audit_ending_decimal,
             'instance_type_id': '1',
             'usage': usage,
             'raw': raw,
@@ -604,14 +611,19 @@ class StacktackUsageParsingTestCase(unittest.TestCase):
         self.mox.VerifyAll()
 
     def test_process_exists_with_deleted_at(self):
-        launch_time = datetime.datetime.utcnow()-datetime.timedelta(hours=23)
-        launch_decimal = utils.decimal_utc(launch_time)
-        deleted_time = datetime.datetime.utcnow()-datetime.timedelta(hours=12)
-        deleted_decimal = utils.decimal_utc(deleted_time)
         current_time = datetime.datetime.utcnow()
+        launch_time = current_time - datetime.timedelta(hours=23)
+        launch_decimal = utils.decimal_utc(launch_time)
+        deleted_time = current_time - datetime.timedelta(hours=12)
+        deleted_decimal = utils.decimal_utc(deleted_time)
         current_decimal = utils.decimal_utc(current_time)
+        audit_beginning = current_time - datetime.timedelta(hours=20)
+        audit_beginning_decimal = utils.decimal_utc(audit_beginning)
+        audit_ending_decimal = utils.decimal_utc(current_time)
         notif = utils.create_nova_notif(launched=str(launch_time),
-                                        deleted=str(deleted_time))
+                                        deleted=str(deleted_time),
+                                        audit_period_beginning=str(audit_beginning),
+                                        audit_period_ending=str(current_time))
         json_str = json.dumps(notif)
         event = 'compute.instance.exists'
         raw = utils.create_raw(self.mox, current_decimal, event=event,
@@ -630,6 +642,8 @@ class StacktackUsageParsingTestCase(unittest.TestCase):
             'instance': INSTANCE_ID_1,
             'launched_at': launch_decimal,
             'deleted_at': deleted_decimal,
+            'audit_period_beginning': audit_beginning_decimal,
+            'audit_period_ending': audit_ending_decimal,
             'instance_type_id': '1',
             'usage': usage,
             'delete': delete,

tests/unit/utils.py

@@ -44,7 +44,8 @@ def decimal_utc(t = datetime.datetime.utcnow()):
 
 def create_nova_notif(request_id=None, instance=INSTANCE_ID_1, type_id='1',
                       launched=None, deleted=None, new_type_id=None,
-                      message_id=MESSAGE_ID_1):
+                      message_id=MESSAGE_ID_1, audit_period_beginning=None,
+                      audit_period_ending=None):
     notif = ['', {
         'message_id': message_id,
         'payload': {
@@ -61,6 +62,10 @@ def create_nova_notif(request_id=None, instance=INSTANCE_ID_1, type_id='1',
         notif[1]['payload']['deleted_at'] = deleted
     if new_type_id:
         notif[1]['payload']['new_instance_type_id'] = new_type_id
+    if audit_period_beginning:
+        notif[1]['payload']['audit_period_beginning'] = audit_period_beginning
+    if audit_period_ending:
+        notif[1]['payload']['audit_period_ending'] = audit_period_ending
 
     return notif