x/stacktach
Code Issues Proposed changes

Adding audit_period_beginning/ending to exists table

Browse Source
This commit is contained in:
Andrew Melton 2013-04-01 14:16:35 -04:00
parent 619ca0ecc7
commit fe27375b82
8 changed files with 257 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
migrations/008_delta.sql Normal file
View File

@ -0,0 +1,2 @@
ALTER TABLE stacktach_instanceexists ADD `audit_period_beginning` numeric(20, 6);
ALTER TABLE stacktach_instanceexists ADD `audit_period_ending` numeric(20, 6);

58
migrations/008_exists_audit_period.py Normal file
View File

@ -0,0 +1,58 @@
import datetime
import os
import sys
try:
import ujson as json
except ImportError:
try:
import simplejson as json
except ImportError:
import json
POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
os.pardir, os.pardir))
if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'stacktach')):
sys.path.insert(0, POSSIBLE_TOPDIR)
from stacktach import models
from stacktach import utils
if __name__ != '__main':
sys.exit(1)
def print_update(total, completed, errored):
to_go = total - (completed + errored)
print "%s populated, %s to go, %s errored" % (completed, to_go, errored)
exists = models.InstanceExists.objects.all()
count = exists.count()
start = datetime.datetime.utcnow()
print "%s records to populate"
update_interval = datetime.timedelta(seconds=30)
next_update = start + update_interval
completed = 0
errored = 0
for exist in exists:
try:
notif = json.loads(exist.raw.json)
payload = notif[1]['payload']
beginning = utils.str_time_to_unix(payload['audit_period_beginning'])
exist.audit_period_beginning = beginning
ending = utils.str_time_to_unix(payload['audit_period_ending'])
exist.audit_period_ending = ending
exist.save()
completed += 1
except:
print "Error with raw %s" % exist.id
errored += 1
if datetime.datetime.utcnow() > next_update:
print_update(count, completed, errored)
next_update = datetime.datetime.utcnow() + update_interval

110
migrations/008_exists_audit_period.sql Normal file
View File

@ -0,0 +1,110 @@
BEGIN;
CREATE TABLE `stacktach_deployment` (
`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
`name` varchar(50) NOT NULL
)
;
CREATE TABLE `stacktach_rawdata` (
`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
`deployment_id` integer NOT NULL,
`tenant` varchar(50),
`json` longtext NOT NULL,
`routing_key` varchar(50),
`state` varchar(20),
`old_state` varchar(20),
`old_task` varchar(30),
`task` varchar(30),
`image_type` integer,
`when` numeric(20, 6) NOT NULL,
`publisher` varchar(100),
`event` varchar(50),
`service` varchar(50),
`host` varchar(100),
`instance` varchar(50),
`request_id` varchar(50)
)
;
ALTER TABLE `stacktach_rawdata` ADD CONSTRAINT `deployment_id_refs_id_362370d` FOREIGN KEY (`deployment_id`) REFERENCES `stacktach_deployment` (`id`);
CREATE TABLE `stacktach_lifecycle` (
`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
`instance` varchar(50),
`last_state` varchar(50),
`last_task_state` varchar(50),
`last_raw_id` integer
)
;
ALTER TABLE `stacktach_lifecycle` ADD CONSTRAINT `last_raw_id_refs_id_d5fb17d3` FOREIGN KEY (`last_raw_id`) REFERENCES `stacktach_rawdata` (`id`);
CREATE TABLE `stacktach_instanceusage` (
`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
`instance` varchar(50),
`launched_at` numeric(20, 6),
`request_id` varchar(50),
`instance_type_id` varchar(50)
)
;
CREATE TABLE `stacktach_instancedeletes` (
`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
`instance` varchar(50),
`launched_at` numeric(20, 6),
`deleted_at` numeric(20, 6),
`raw_id` integer
)
;
ALTER TABLE `stacktach_instancedeletes` ADD CONSTRAINT `raw_id_refs_id_58031c62` FOREIGN KEY (`raw_id`) REFERENCES `stacktach_rawdata` (`id`);
CREATE TABLE `stacktach_instanceexists` (
`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
`instance` varchar(50),
`launched_at` numeric(20, 6),
`deleted_at` numeric(20, 6),
`audit_period_beginning` numeric(20, 6),
`audit_period_ending` numeric(20, 6),
`message_id` varchar(50),
`instance_type_id` varchar(50),
`status` varchar(50) NOT NULL,
`fail_reason` varchar(500),
`raw_id` integer,
`usage_id` integer,
`delete_id` integer,
`send_status` integer
)
;
ALTER TABLE `stacktach_instanceexists` ADD CONSTRAINT `delete_id_refs_id_e02dfe84` FOREIGN KEY (`delete_id`) REFERENCES `stacktach_instancedeletes` (`id`);
ALTER TABLE `stacktach_instanceexists` ADD CONSTRAINT `usage_id_refs_id_c4ecd665` FOREIGN KEY (`usage_id`) REFERENCES `stacktach_instanceusage` (`id`);
ALTER TABLE `stacktach_instanceexists` ADD CONSTRAINT `raw_id_refs_id_65c72953` FOREIGN KEY (`raw_id`) REFERENCES `stacktach_rawdata` (`id`);
CREATE TABLE `stacktach_timing` (
`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
`name` varchar(50) NOT NULL,
`lifecycle_id` integer NOT NULL,
`start_raw_id` integer,
`end_raw_id` integer,
`start_when` numeric(20, 6),
`end_when` numeric(20, 6),
`diff` numeric(20, 6)
)
;
ALTER TABLE `stacktach_timing` ADD CONSTRAINT `start_raw_id_refs_id_c32dfe04` FOREIGN KEY (`start_raw_id`) REFERENCES `stacktach_rawdata` (`id`);
ALTER TABLE `stacktach_timing` ADD CONSTRAINT `end_raw_id_refs_id_c32dfe04` FOREIGN KEY (`end_raw_id`) REFERENCES `stacktach_rawdata` (`id`);
ALTER TABLE `stacktach_timing` ADD CONSTRAINT `lifecycle_id_refs_id_4255ead8` FOREIGN KEY (`lifecycle_id`) REFERENCES `stacktach_lifecycle` (`id`);
CREATE TABLE `stacktach_requesttracker` (
`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
`request_id` varchar(50) NOT NULL,
`lifecycle_id` integer NOT NULL,
`last_timing_id` integer,
`start` numeric(20, 6) NOT NULL,
`duration` numeric(20, 6) NOT NULL,
`completed` bool NOT NULL
)
;
ALTER TABLE `stacktach_requesttracker` ADD CONSTRAINT `last_timing_id_refs_id_f0827cca` FOREIGN KEY (`last_timing_id`) REFERENCES `stacktach_timing` (`id`);
ALTER TABLE `stacktach_requesttracker` ADD CONSTRAINT `lifecycle_id_refs_id_e457729` FOREIGN KEY (`lifecycle_id`) REFERENCES `stacktach_lifecycle` (`id`);
CREATE TABLE `stacktach_jsonreport` (
`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
`period_start` datetime NOT NULL,
`period_end` datetime NOT NULL,
`created` numeric(20, 6) NOT NULL,
`name` varchar(50) NOT NULL,
`version` integer NOT NULL,
`json` longtext NOT NULL
)
;
COMMIT;

50
migrations/008_exists_audit_period_indexes.sql Normal file
View File

@ -0,0 +1,50 @@
BEGIN;
CREATE INDEX `stacktach_rawdata_4ac6801` ON `stacktach_rawdata` (`deployment_id`);
CREATE INDEX `stacktach_rawdata_2207f86d` ON `stacktach_rawdata` (`tenant`);
CREATE INDEX `stacktach_rawdata_2192f43a` ON `stacktach_rawdata` (`routing_key`);
CREATE INDEX `stacktach_rawdata_355bfc27` ON `stacktach_rawdata` (`state`);
CREATE INDEX `stacktach_rawdata_b716e0bb` ON `stacktach_rawdata` (`old_state`);
CREATE INDEX `stacktach_rawdata_8182be12` ON `stacktach_rawdata` (`old_task`);
CREATE INDEX `stacktach_rawdata_1c149b74` ON `stacktach_rawdata` (`task`);
CREATE INDEX `stacktach_rawdata_cfde77eb` ON `stacktach_rawdata` (`image_type`);
CREATE INDEX `stacktach_rawdata_feaed089` ON `stacktach_rawdata` (`when`);
CREATE INDEX `stacktach_rawdata_878a2906` ON `stacktach_rawdata` (`publisher`);
CREATE INDEX `stacktach_rawdata_a90f9116` ON `stacktach_rawdata` (`event`);
CREATE INDEX `stacktach_rawdata_52c5ef6b` ON `stacktach_rawdata` (`service`);
CREATE INDEX `stacktach_rawdata_38dbea87` ON `stacktach_rawdata` (`host`);
CREATE INDEX `stacktach_rawdata_888b756a` ON `stacktach_rawdata` (`instance`);
CREATE INDEX `stacktach_rawdata_792812e8` ON `stacktach_rawdata` (`request_id`);
CREATE INDEX `stacktach_lifecycle_888b756a` ON `stacktach_lifecycle` (`instance`);
CREATE INDEX `stacktach_lifecycle_9b2555fd` ON `stacktach_lifecycle` (`last_state`);
CREATE INDEX `stacktach_lifecycle_67421a0e` ON `stacktach_lifecycle` (`last_task_state`);
CREATE INDEX `stacktach_lifecycle_dcf9e5f3` ON `stacktach_lifecycle` (`last_raw_id`);
CREATE INDEX `stacktach_instanceusage_888b756a` ON `stacktach_instanceusage` (`instance`);
CREATE INDEX `stacktach_instanceusage_792812e8` ON `stacktach_instanceusage` (`request_id`);
CREATE INDEX `stacktach_instanceusage_f321fd7` ON `stacktach_instanceusage` (`instance_type_id`);
CREATE INDEX `stacktach_instancedeletes_888b756a` ON `stacktach_instancedeletes` (`instance`);
CREATE INDEX `stacktach_instancedeletes_365c3a01` ON `stacktach_instancedeletes` (`raw_id`);
CREATE INDEX `stacktach_instanceexists_888b756a` ON `stacktach_instanceexists` (`instance`);
CREATE INDEX `stacktach_instanceexists_38373776` ON `stacktach_instanceexists` (`message_id`);
CREATE INDEX `stacktach_instanceexists_f321fd7` ON `stacktach_instanceexists` (`instance_type_id`);
CREATE INDEX `stacktach_instanceexists_c9ad71dd` ON `stacktach_instanceexists` (`status`);
CREATE INDEX `stacktach_instanceexists_347f3d31` ON `stacktach_instanceexists` (`fail_reason`);
CREATE INDEX `stacktach_instanceexists_365c3a01` ON `stacktach_instanceexists` (`raw_id`);
CREATE INDEX `stacktach_instanceexists_d9ffa990` ON `stacktach_instanceexists` (`usage_id`);
CREATE INDEX `stacktach_instanceexists_cb6f05a7` ON `stacktach_instanceexists` (`delete_id`);
CREATE INDEX `stacktach_instanceexists_b2444339` ON `stacktach_instanceexists` (`send_status`);
CREATE INDEX `stacktach_timing_52094d6e` ON `stacktach_timing` (`name`);
CREATE INDEX `stacktach_timing_9f222e6b` ON `stacktach_timing` (`lifecycle_id`);
CREATE INDEX `stacktach_timing_efab905a` ON `stacktach_timing` (`start_raw_id`);
CREATE INDEX `stacktach_timing_c8bb8daf` ON `stacktach_timing` (`end_raw_id`);
CREATE INDEX `stacktach_timing_4401d15e` ON `stacktach_timing` (`diff`);
CREATE INDEX `stacktach_requesttracker_792812e8` ON `stacktach_requesttracker` (`request_id`);
CREATE INDEX `stacktach_requesttracker_9f222e6b` ON `stacktach_requesttracker` (`lifecycle_id`);
CREATE INDEX `stacktach_requesttracker_ce616a96` ON `stacktach_requesttracker` (`last_timing_id`);
CREATE INDEX `stacktach_requesttracker_29f4f2ea` ON `stacktach_requesttracker` (`start`);
CREATE INDEX `stacktach_requesttracker_8eb45f9b` ON `stacktach_requesttracker` (`duration`);
CREATE INDEX `stacktach_requesttracker_e490d511` ON `stacktach_requesttracker` (`completed`);
CREATE INDEX `stacktach_jsonreport_70ecb89f` ON `stacktach_jsonreport` (`period_start`);
CREATE INDEX `stacktach_jsonreport_6a26a758` ON `stacktach_jsonreport` (`period_end`);
CREATE INDEX `stacktach_jsonreport_3216ff68` ON `stacktach_jsonreport` (`created`);
CREATE INDEX `stacktach_jsonreport_52094d6e` ON `stacktach_jsonreport` (`name`);
COMMIT;

4
stacktach/models.py
View File

@ -115,6 +115,10 @@ class InstanceExists(models.Model):
decimal_places=6)
deleted_at = models.DecimalField(null=True, max_digits=20,
decimal_places=6)
audit_period_beginning = models.DecimalField(null=True, max_digits=20,
decimal_places=6)
audit_period_ending = models.DecimalField(null=True, max_digits=20,
decimal_places=6)
message_id = models.CharField(max_length=50, null=True,
blank=True, db_index=True)
instance_type_id = models.CharField(max_length=50,

4
stacktach/views.py
View File

@ -286,6 +286,10 @@ def _process_exists(raw, body):
values['message_id'] = body['message_id']
values['instance'] = instance_id
values['launched_at'] = launched_at
beginning = utils.str_time_to_unix(payload['audit_period_beginning'])
values['audit_period_beginning'] = beginning
ending = utils.str_time_to_unix(payload['audit_period_ending'])
values['audit_period_ending'] = ending
values['instance_type_id'] = payload['instance_type_id']
if usage:
values['usage'] = usage

30
tests/unit/test_stacktach.py
View File

@ -571,11 +571,16 @@ class StacktackUsageParsingTestCase(unittest.TestCase):
self.mox.VerifyAll()
def test_process_exists(self):
launch_time = datetime.datetime.utcnow()-datetime.timedelta(hours=23)
launch_decimal = utils.decimal_utc(launch_time)
current_time = datetime.datetime.utcnow()
launch_time = current_time - datetime.timedelta(hours=23)
launch_decimal = utils.decimal_utc(launch_time)
current_decimal = utils.decimal_utc(current_time)
notif = utils.create_nova_notif(launched=str(launch_time))
audit_beginning = current_time - datetime.timedelta(hours=20)
audit_beginning_decimal = utils.decimal_utc(audit_beginning)
audit_ending_decimal = utils.decimal_utc(current_time)
notif = utils.create_nova_notif(launched=str(launch_time),
audit_period_beginning=str(audit_beginning),
audit_period_ending=str(current_time))
json_str = json.dumps(notif)
event = 'compute.instance.exists'
raw = utils.create_raw(self.mox, current_decimal, event=event,
@ -592,6 +597,8 @@ class StacktackUsageParsingTestCase(unittest.TestCase):
'message_id': MESSAGE_ID_1,
'instance': INSTANCE_ID_1,
'launched_at': launch_decimal,
'audit_period_beginning': audit_beginning_decimal,
'audit_period_ending': audit_ending_decimal,
'instance_type_id': '1',
'usage': usage,
'raw': raw,
@ -604,14 +611,19 @@ class StacktackUsageParsingTestCase(unittest.TestCase):
self.mox.VerifyAll()
def test_process_exists_with_deleted_at(self):
launch_time = datetime.datetime.utcnow()-datetime.timedelta(hours=23)
launch_decimal = utils.decimal_utc(launch_time)
deleted_time = datetime.datetime.utcnow()-datetime.timedelta(hours=12)
deleted_decimal = utils.decimal_utc(deleted_time)
current_time = datetime.datetime.utcnow()
launch_time = current_time - datetime.timedelta(hours=23)
launch_decimal = utils.decimal_utc(launch_time)
deleted_time = current_time - datetime.timedelta(hours=12)
deleted_decimal = utils.decimal_utc(deleted_time)
current_decimal = utils.decimal_utc(current_time)
audit_beginning = current_time - datetime.timedelta(hours=20)
audit_beginning_decimal = utils.decimal_utc(audit_beginning)
audit_ending_decimal = utils.decimal_utc(current_time)
notif = utils.create_nova_notif(launched=str(launch_time),
deleted=str(deleted_time))
deleted=str(deleted_time),
audit_period_beginning=str(audit_beginning),
audit_period_ending=str(current_time))
json_str = json.dumps(notif)
event = 'compute.instance.exists'
raw = utils.create_raw(self.mox, current_decimal, event=event,
@ -630,6 +642,8 @@ class StacktackUsageParsingTestCase(unittest.TestCase):
'instance': INSTANCE_ID_1,
'launched_at': launch_decimal,
'deleted_at': deleted_decimal,
'audit_period_beginning': audit_beginning_decimal,
'audit_period_ending': audit_ending_decimal,
'instance_type_id': '1',
'usage': usage,
'delete': delete,

9
tests/unit/utils.py
View File

@ -43,8 +43,9 @@ def decimal_utc(t = datetime.datetime.utcnow()):
def create_nova_notif(request_id=None, instance=INSTANCE_ID_1, type_id='1',
launched=None, deleted = None, new_type_id=None,
message_id=MESSAGE_ID_1):
launched=None, deleted=None, new_type_id=None,
message_id=MESSAGE_ID_1, audit_period_beginning=None,
audit_period_ending=None):
notif = ['', {
'message_id': message_id,
'payload': {
@ -61,6 +62,10 @@ def create_nova_notif(request_id=None, instance=INSTANCE_ID_1, type_id='1',
notif[1]['payload']['deleted_at'] = deleted
if new_type_id:
notif[1]['payload']['new_instance_type_id'] = new_type_id
if audit_period_beginning:
notif[1]['payload']['audit_period_beginning'] = audit_period_beginning
if audit_period_ending:
notif[1]['payload']['audit_period_ending'] = audit_period_ending
return notif