x/stacktach
Code Issues Proposed changes

Merge branch 'master' of git://github.com/rackerlabs/stacktach into auditor_cron

Browse Source
This commit is contained in:
Andrew Melton 2013-04-01 16:48:30 -04:00
commit 592abd9148
6 changed files with 49 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
migrations/006_populate_rebuilds_from_rawdata.py
View File

@ -58,8 +58,9 @@ def add_past_usage(raws):
if processed % 50 == 0:
next_update = last_update + datetime.timedelta(seconds=30)
if datetime.datetime.utcnow() > next_update:
(processed, count - processed, float(processed) / count)
print "%s processed, %s to go, %.2f percent done"
m = (processed, count - processed, float(processed) / count)
print "%s processed, %s to go, %.2f percent done" % m
last_update = datetime.datetime.utcnow()
print "completed processing %s events" % count

9
migrations/008_delta.sql
View File

@ -1,2 +1,9 @@
ALTER TABLE stacktach_instanceexists ADD `audit_period_beginning` numeric(20, 6);
ALTER TABLE stacktach_instanceexists ADD `audit_period_ending` numeric(20, 6);
ALTER TABLE stacktach_instanceexists ADD `audit_period_ending` numeric(20, 6);
CREATE INDEX `stacktach_instanceusage_987c9676` ON `stacktach_instanceusage` (`launched_at`);
CREATE INDEX `stacktach_instancedeletes_987c9676` ON `stacktach_instancedeletes` (`launched_at`);
CREATE INDEX `stacktach_instancedeletes_738c7e64` ON `stacktach_instancedeletes` (`deleted_at`);
CREATE INDEX `stacktach_instanceexists_987c9676` ON `stacktach_instanceexists` (`launched_at`);
CREATE INDEX `stacktach_instanceexists_738c7e64` ON `stacktach_instanceexists` (`deleted_at`);
CREATE INDEX `stacktach_instanceexists_23564986` ON `stacktach_instanceexists` (`audit_period_beginning`);
CREATE INDEX `stacktach_instanceexists_b891fefb` ON `stacktach_instanceexists` (`audit_period_ending`);

7
migrations/008_exists_audit_period_indexes.sql
View File

@ -19,11 +19,18 @@ CREATE INDEX `stacktach_lifecycle_9b2555fd` ON `stacktach_lifecycle` (`last_stat
CREATE INDEX `stacktach_lifecycle_67421a0e` ON `stacktach_lifecycle` (`last_task_state`);
CREATE INDEX `stacktach_lifecycle_dcf9e5f3` ON `stacktach_lifecycle` (`last_raw_id`);
CREATE INDEX `stacktach_instanceusage_888b756a` ON `stacktach_instanceusage` (`instance`);
CREATE INDEX `stacktach_instanceusage_987c9676` ON `stacktach_instanceusage` (`launched_at`);
CREATE INDEX `stacktach_instanceusage_792812e8` ON `stacktach_instanceusage` (`request_id`);
CREATE INDEX `stacktach_instanceusage_f321fd7` ON `stacktach_instanceusage` (`instance_type_id`);
CREATE INDEX `stacktach_instancedeletes_888b756a` ON `stacktach_instancedeletes` (`instance`);
CREATE INDEX `stacktach_instancedeletes_987c9676` ON `stacktach_instancedeletes` (`launched_at`);
CREATE INDEX `stacktach_instancedeletes_738c7e64` ON `stacktach_instancedeletes` (`deleted_at`);
CREATE INDEX `stacktach_instancedeletes_365c3a01` ON `stacktach_instancedeletes` (`raw_id`);
CREATE INDEX `stacktach_instanceexists_888b756a` ON `stacktach_instanceexists` (`instance`);
CREATE INDEX `stacktach_instanceexists_987c9676` ON `stacktach_instanceexists` (`launched_at`);
CREATE INDEX `stacktach_instanceexists_738c7e64` ON `stacktach_instanceexists` (`deleted_at`);
CREATE INDEX `stacktach_instanceexists_23564986` ON `stacktach_instanceexists` (`audit_period_beginning`);
CREATE INDEX `stacktach_instanceexists_b891fefb` ON `stacktach_instanceexists` (`audit_period_ending`);
CREATE INDEX `stacktach_instanceexists_38373776` ON `stacktach_instanceexists` (`message_id`);
CREATE INDEX `stacktach_instanceexists_f321fd7` ON `stacktach_instanceexists` (`instance_type_id`);
CREATE INDEX `stacktach_instanceexists_c9ad71dd` ON `stacktach_instanceexists` (`status`);

7
migrations/009_delta.sql
View File

@ -1,7 +0,0 @@
CREATE INDEX `stacktach_instanceusage_987c9676` ON `stacktach_instanceusage` (`launched_at`);
CREATE INDEX `stacktach_instancedeletes_987c9676` ON `stacktach_instancedeletes` (`launched_at`);
CREATE INDEX `stacktach_instancedeletes_738c7e64` ON `stacktach_instancedeletes` (`deleted_at`);
CREATE INDEX `stacktach_instanceexists_987c9676` ON `stacktach_instanceexists` (`launched_at`);
CREATE INDEX `stacktach_instanceexists_738c7e64` ON `stacktach_instanceexists` (`deleted_at`);
CREATE INDEX `stacktach_instanceexists_23564986` ON `stacktach_instanceexists` (`audit_period_beginning`);
CREATE INDEX `stacktach_instanceexists_b891fefb` ON `stacktach_instanceexists` (`audit_period_ending`);

57
migrations/009_usage_missing_indexes.sql
View File

@ -1,57 +0,0 @@
BEGIN;
CREATE INDEX `stacktach_rawdata_4ac6801` ON `stacktach_rawdata` (`deployment_id`);
CREATE INDEX `stacktach_rawdata_2207f86d` ON `stacktach_rawdata` (`tenant`);
CREATE INDEX `stacktach_rawdata_2192f43a` ON `stacktach_rawdata` (`routing_key`);
CREATE INDEX `stacktach_rawdata_355bfc27` ON `stacktach_rawdata` (`state`);
CREATE INDEX `stacktach_rawdata_b716e0bb` ON `stacktach_rawdata` (`old_state`);
CREATE INDEX `stacktach_rawdata_8182be12` ON `stacktach_rawdata` (`old_task`);
CREATE INDEX `stacktach_rawdata_1c149b74` ON `stacktach_rawdata` (`task`);
CREATE INDEX `stacktach_rawdata_cfde77eb` ON `stacktach_rawdata` (`image_type`);
CREATE INDEX `stacktach_rawdata_feaed089` ON `stacktach_rawdata` (`when`);
CREATE INDEX `stacktach_rawdata_878a2906` ON `stacktach_rawdata` (`publisher`);
CREATE INDEX `stacktach_rawdata_a90f9116` ON `stacktach_rawdata` (`event`);
CREATE INDEX `stacktach_rawdata_52c5ef6b` ON `stacktach_rawdata` (`service`);
CREATE INDEX `stacktach_rawdata_38dbea87` ON `stacktach_rawdata` (`host`);
CREATE INDEX `stacktach_rawdata_888b756a` ON `stacktach_rawdata` (`instance`);
CREATE INDEX `stacktach_rawdata_792812e8` ON `stacktach_rawdata` (`request_id`);
CREATE INDEX `stacktach_lifecycle_888b756a` ON `stacktach_lifecycle` (`instance`);
CREATE INDEX `stacktach_lifecycle_9b2555fd` ON `stacktach_lifecycle` (`last_state`);
CREATE INDEX `stacktach_lifecycle_67421a0e` ON `stacktach_lifecycle` (`last_task_state`);
CREATE INDEX `stacktach_lifecycle_dcf9e5f3` ON `stacktach_lifecycle` (`last_raw_id`);
CREATE INDEX `stacktach_instanceusage_888b756a` ON `stacktach_instanceusage` (`instance`);
CREATE INDEX `stacktach_instanceusage_987c9676` ON `stacktach_instanceusage` (`launched_at`);
CREATE INDEX `stacktach_instanceusage_792812e8` ON `stacktach_instanceusage` (`request_id`);
CREATE INDEX `stacktach_instanceusage_f321fd7` ON `stacktach_instanceusage` (`instance_type_id`);
CREATE INDEX `stacktach_instancedeletes_888b756a` ON `stacktach_instancedeletes` (`instance`);
CREATE INDEX `stacktach_instancedeletes_987c9676` ON `stacktach_instancedeletes` (`launched_at`);
CREATE INDEX `stacktach_instancedeletes_738c7e64` ON `stacktach_instancedeletes` (`deleted_at`);
CREATE INDEX `stacktach_instancedeletes_365c3a01` ON `stacktach_instancedeletes` (`raw_id`);
CREATE INDEX `stacktach_instanceexists_888b756a` ON `stacktach_instanceexists` (`instance`);
CREATE INDEX `stacktach_instanceexists_987c9676` ON `stacktach_instanceexists` (`launched_at`);
CREATE INDEX `stacktach_instanceexists_738c7e64` ON `stacktach_instanceexists` (`deleted_at`);
CREATE INDEX `stacktach_instanceexists_23564986` ON `stacktach_instanceexists` (`audit_period_beginning`);
CREATE INDEX `stacktach_instanceexists_b891fefb` ON `stacktach_instanceexists` (`audit_period_ending`);
CREATE INDEX `stacktach_instanceexists_38373776` ON `stacktach_instanceexists` (`message_id`);
CREATE INDEX `stacktach_instanceexists_f321fd7` ON `stacktach_instanceexists` (`instance_type_id`);
CREATE INDEX `stacktach_instanceexists_c9ad71dd` ON `stacktach_instanceexists` (`status`);
CREATE INDEX `stacktach_instanceexists_347f3d31` ON `stacktach_instanceexists` (`fail_reason`);
CREATE INDEX `stacktach_instanceexists_365c3a01` ON `stacktach_instanceexists` (`raw_id`);
CREATE INDEX `stacktach_instanceexists_d9ffa990` ON `stacktach_instanceexists` (`usage_id`);
CREATE INDEX `stacktach_instanceexists_cb6f05a7` ON `stacktach_instanceexists` (`delete_id`);
CREATE INDEX `stacktach_instanceexists_b2444339` ON `stacktach_instanceexists` (`send_status`);
CREATE INDEX `stacktach_timing_52094d6e` ON `stacktach_timing` (`name`);
CREATE INDEX `stacktach_timing_9f222e6b` ON `stacktach_timing` (`lifecycle_id`);
CREATE INDEX `stacktach_timing_efab905a` ON `stacktach_timing` (`start_raw_id`);
CREATE INDEX `stacktach_timing_c8bb8daf` ON `stacktach_timing` (`end_raw_id`);
CREATE INDEX `stacktach_timing_4401d15e` ON `stacktach_timing` (`diff`);
CREATE INDEX `stacktach_requesttracker_792812e8` ON `stacktach_requesttracker` (`request_id`);
CREATE INDEX `stacktach_requesttracker_9f222e6b` ON `stacktach_requesttracker` (`lifecycle_id`);
CREATE INDEX `stacktach_requesttracker_ce616a96` ON `stacktach_requesttracker` (`last_timing_id`);
CREATE INDEX `stacktach_requesttracker_29f4f2ea` ON `stacktach_requesttracker` (`start`);
CREATE INDEX `stacktach_requesttracker_8eb45f9b` ON `stacktach_requesttracker` (`duration`);
CREATE INDEX `stacktach_requesttracker_e490d511` ON `stacktach_requesttracker` (`completed`);
CREATE INDEX `stacktach_jsonreport_70ecb89f` ON `stacktach_jsonreport` (`period_start`);
CREATE INDEX `stacktach_jsonreport_6a26a758` ON `stacktach_jsonreport` (`period_end`);
CREATE INDEX `stacktach_jsonreport_3216ff68` ON `stacktach_jsonreport` (`created`);
CREATE INDEX `stacktach_jsonreport_52094d6e` ON `stacktach_jsonreport` (`name`);
COMMIT;

41
reports/nova_usage_audit.py
View File

@ -25,6 +25,8 @@ sys.path.append("/stacktach")
from stacktach import datetime_to_decimal as dt
from stacktach import models
OLD_LAUNCHES_QUERY = "select * from stacktach_instanceusage where launched_at is not null and launched_at < %s and instance not in (select distinct(instance) from stacktach_instancedeletes where deleted_at < %s)"
def _get_new_launches(beginning, ending):
filters = {
@ -73,23 +75,23 @@ def _audit_launches_to_exists(launches, exists):
return fails
def _audit_exists_to_launches(exists, launches):
def _audit_launches_to_deletes(deletes, exists):
fails = []
for (instance, rows) in exists.items():
if instance in launches:
for exist in rows:
for (instance, delete_list) in deletes.items():
if instance in exists:
for delete in delete_list:
found = False
for launch in exists[instance]:
if int(exist['launched_at']) == int(launch['launched_at']):
for exist in exists[instance]:
if int(delete['deleted_at']) == int(exist['deleted_at']):
# HACK (apmelton): Truncate the decimal because we may not
# have the milliseconds.
found = True
if not found:
msg = "Couldn't find exists for launch (%s, %s)"
fails.append(msg % (instance, exist['launched_at']))
msg = "Couldn't find exists for delete (%s, %s)"
fails.append(msg % (instance, delete['deleted_at']))
else:
msg = "No launch for instance (%s)" % instance
msg = "No exists for instance (%s)" % instance
fails.append(msg)
return fails
@ -105,12 +107,27 @@ def _audit_for_period(beginning, ending):
else:
launches_dict[instance] = [l, ]
old_launches = models.InstanceUsage.objects.raw(OLD_LAUNCHES_QUERY,
[beginning, beginning])
old_launches_dict = {}
for launch in old_launches:
instance = launch.instance
l = {'id': launch.id, 'launched_at': launch.launched_at}
if instance not in launches_dict or \
launches_dict[instance] < launch.launched_at:
old_launches_dict[instance] = l
for instance, launch in old_launches_dict.items():
if instance in launches_dict:
launches_dict[instance].append(launch)
else:
launches_dict[instance] = [launch, ]
deletes_dict = {}
deletes = _get_deletes(beginning, ending)
for delete in deletes:
instance = delete.instance
d = {'id': delete.id,
'launched_at': delete.launched_at,
'deleted_at': delete.deleted_at}
if instance in deletes_dict:
deletes_dict[instance].append(d)
@ -131,6 +148,10 @@ def _audit_for_period(beginning, ending):
launch_to_exists_fails = _audit_launches_to_exists(launches_dict,
exists_dict)
delete_to_exists_fails = _audit_launches_to_deletes(deletes_dict,
exists_dict)
return launch_to_exists_fails, delete_to_exists_fails
def audit_for_period(beginning, ending):