x/stacktach
Code Issues Proposed changes

Support for task state, image_type and pretty summary report

Browse Source
This commit is contained in:
Sandy Walsh 2013-02-13 11:31:21 -04:00
commit a5123d916d
15 changed files with 850 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
migrations/001_base.sql Normal file
View File

@ -0,0 +1,61 @@
BEGIN;
CREATE TABLE `stacktach_deployment` (
`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
`name` varchar(50) NOT NULL
)
;
CREATE TABLE `stacktach_rawdata` (
`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
`deployment_id` integer NOT NULL,
`tenant` varchar(50),
`json` longtext NOT NULL,
`routing_key` varchar(50),
`state` varchar(20),
`old_state` varchar(20),
`old_task` varchar(30),
`when` numeric(20, 6) NOT NULL,
`publisher` varchar(100),
`event` varchar(50),
`service` varchar(50),
`host` varchar(100),
`instance` varchar(50),
`request_id` varchar(50)
)
;
ALTER TABLE `stacktach_rawdata` ADD CONSTRAINT `deployment_id_refs_id_362370d` FOREIGN KEY (`deployment_id`) REFERENCES `stacktach_deployment` (`id`);
CREATE TABLE `stacktach_lifecycle` (
`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
`instance` varchar(50),
`last_state` varchar(50),
`last_task_state` varchar(50),
`last_raw_id` integer
)
;
ALTER TABLE `stacktach_lifecycle` ADD CONSTRAINT `last_raw_id_refs_id_d5fb17d3` FOREIGN KEY (`last_raw_id`) REFERENCES `stacktach_rawdata` (`id`);
CREATE TABLE `stacktach_timing` (
`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
`name` varchar(50) NOT NULL,
`lifecycle_id` integer NOT NULL,
`start_raw_id` integer,
`end_raw_id` integer,
`start_when` numeric(20, 6),
`end_when` numeric(20, 6),
`diff` numeric(20, 6)
)
;
ALTER TABLE `stacktach_timing` ADD CONSTRAINT `lifecycle_id_refs_id_4255ead8` FOREIGN KEY (`lifecycle_id`) REFERENCES `stacktach_lifecycle` (`id`);
ALTER TABLE `stacktach_timing` ADD CONSTRAINT `start_raw_id_refs_id_c32dfe04` FOREIGN KEY (`start_raw_id`) REFERENCES `stacktach_rawdata` (`id`);
ALTER TABLE `stacktach_timing` ADD CONSTRAINT `end_raw_id_refs_id_c32dfe04` FOREIGN KEY (`end_raw_id`) REFERENCES `stacktach_rawdata` (`id`);
CREATE TABLE `stacktach_requesttracker` (
`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
`request_id` varchar(50) NOT NULL,
`lifecycle_id` integer NOT NULL,
`last_timing_id` integer,
`start` numeric(20, 6) NOT NULL,
`duration` numeric(20, 6) NOT NULL,
`completed` bool NOT NULL
)
;
ALTER TABLE `stacktach_requesttracker` ADD CONSTRAINT `lifecycle_id_refs_id_e457729` FOREIGN KEY (`lifecycle_id`) REFERENCES `stacktach_lifecycle` (`id`);
ALTER TABLE `stacktach_requesttracker` ADD CONSTRAINT `last_timing_id_refs_id_f0827cca` FOREIGN KEY (`last_timing_id`) REFERENCES `stacktach_timing` (`id`);
COMMIT;

2
migrations/002_delta.sql Normal file
View File

@ -0,0 +1,2 @@
ALTER TABLE stacktach_rawdata ADD task VARCHAR(30);
CREATE INDEX `stacktach_rawdata_1c149b74` ON `stacktach_rawdata` (`task`);

62
migrations/002_task.sql Normal file
View File

@ -0,0 +1,62 @@
BEGIN;
CREATE TABLE `stacktach_deployment` (
`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
`name` varchar(50) NOT NULL
)
;
CREATE TABLE `stacktach_rawdata` (
`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
`deployment_id` integer NOT NULL,
`tenant` varchar(50),
`json` longtext NOT NULL,
`routing_key` varchar(50),
`state` varchar(20),
`old_state` varchar(20),
`old_task` varchar(30),
`task` varchar(30),
`when` numeric(20, 6) NOT NULL,
`publisher` varchar(100),
`event` varchar(50),
`service` varchar(50),
`host` varchar(100),
`instance` varchar(50),
`request_id` varchar(50)
)
;
ALTER TABLE `stacktach_rawdata` ADD CONSTRAINT `deployment_id_refs_id_362370d` FOREIGN KEY (`deployment_id`) REFERENCES `stacktach_deployment` (`id`);
CREATE TABLE `stacktach_lifecycle` (
`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
`instance` varchar(50),
`last_state` varchar(50),
`last_task_state` varchar(50),
`last_raw_id` integer
)
;
ALTER TABLE `stacktach_lifecycle` ADD CONSTRAINT `last_raw_id_refs_id_d5fb17d3` FOREIGN KEY (`last_raw_id`) REFERENCES `stacktach_rawdata` (`id`);
CREATE TABLE `stacktach_timing` (
`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
`name` varchar(50) NOT NULL,
`lifecycle_id` integer NOT NULL,
`start_raw_id` integer,
`end_raw_id` integer,
`start_when` numeric(20, 6),
`end_when` numeric(20, 6),
`diff` numeric(20, 6)
)
;
ALTER TABLE `stacktach_timing` ADD CONSTRAINT `lifecycle_id_refs_id_4255ead8` FOREIGN KEY (`lifecycle_id`) REFERENCES `stacktach_lifecycle` (`id`);
ALTER TABLE `stacktach_timing` ADD CONSTRAINT `start_raw_id_refs_id_c32dfe04` FOREIGN KEY (`start_raw_id`) REFERENCES `stacktach_rawdata` (`id`);
ALTER TABLE `stacktach_timing` ADD CONSTRAINT `end_raw_id_refs_id_c32dfe04` FOREIGN KEY (`end_raw_id`) REFERENCES `stacktach_rawdata` (`id`);
CREATE TABLE `stacktach_requesttracker` (
`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
`request_id` varchar(50) NOT NULL,
`lifecycle_id` integer NOT NULL,
`last_timing_id` integer,
`start` numeric(20, 6) NOT NULL,
`duration` numeric(20, 6) NOT NULL,
`completed` bool NOT NULL
)
;
ALTER TABLE `stacktach_requesttracker` ADD CONSTRAINT `lifecycle_id_refs_id_e457729` FOREIGN KEY (`lifecycle_id`) REFERENCES `stacktach_lifecycle` (`id`);
ALTER TABLE `stacktach_requesttracker` ADD CONSTRAINT `last_timing_id_refs_id_f0827cca` FOREIGN KEY (`last_timing_id`) REFERENCES `stacktach_timing` (`id`);
COMMIT;

31
migrations/002_task_indexes.sql Normal file
View File

@ -0,0 +1,31 @@
BEGIN;
CREATE INDEX `stacktach_rawdata_4ac6801` ON `stacktach_rawdata` (`deployment_id`);
CREATE INDEX `stacktach_rawdata_2207f86d` ON `stacktach_rawdata` (`tenant`);
CREATE INDEX `stacktach_rawdata_2192f43a` ON `stacktach_rawdata` (`routing_key`);
CREATE INDEX `stacktach_rawdata_355bfc27` ON `stacktach_rawdata` (`state`);
CREATE INDEX `stacktach_rawdata_b716e0bb` ON `stacktach_rawdata` (`old_state`);
CREATE INDEX `stacktach_rawdata_8182be12` ON `stacktach_rawdata` (`old_task`);
CREATE INDEX `stacktach_rawdata_1c149b74` ON `stacktach_rawdata` (`task`);
CREATE INDEX `stacktach_rawdata_feaed089` ON `stacktach_rawdata` (`when`);
CREATE INDEX `stacktach_rawdata_878a2906` ON `stacktach_rawdata` (`publisher`);
CREATE INDEX `stacktach_rawdata_a90f9116` ON `stacktach_rawdata` (`event`);
CREATE INDEX `stacktach_rawdata_52c5ef6b` ON `stacktach_rawdata` (`service`);
CREATE INDEX `stacktach_rawdata_38dbea87` ON `stacktach_rawdata` (`host`);
CREATE INDEX `stacktach_rawdata_888b756a` ON `stacktach_rawdata` (`instance`);
CREATE INDEX `stacktach_rawdata_792812e8` ON `stacktach_rawdata` (`request_id`);
CREATE INDEX `stacktach_lifecycle_888b756a` ON `stacktach_lifecycle` (`instance`);
CREATE INDEX `stacktach_lifecycle_9b2555fd` ON `stacktach_lifecycle` (`last_state`);
CREATE INDEX `stacktach_lifecycle_67421a0e` ON `stacktach_lifecycle` (`last_task_state`);
CREATE INDEX `stacktach_lifecycle_dcf9e5f3` ON `stacktach_lifecycle` (`last_raw_id`);
CREATE INDEX `stacktach_timing_52094d6e` ON `stacktach_timing` (`name`);
CREATE INDEX `stacktach_timing_9f222e6b` ON `stacktach_timing` (`lifecycle_id`);
CREATE INDEX `stacktach_timing_efab905a` ON `stacktach_timing` (`start_raw_id`);
CREATE INDEX `stacktach_timing_c8bb8daf` ON `stacktach_timing` (`end_raw_id`);
CREATE INDEX `stacktach_timing_4401d15e` ON `stacktach_timing` (`diff`);
CREATE INDEX `stacktach_requesttracker_792812e8` ON `stacktach_requesttracker` (`request_id`);
CREATE INDEX `stacktach_requesttracker_9f222e6b` ON `stacktach_requesttracker` (`lifecycle_id`);
CREATE INDEX `stacktach_requesttracker_ce616a96` ON `stacktach_requesttracker` (`last_timing_id`);
CREATE INDEX `stacktach_requesttracker_29f4f2ea` ON `stacktach_requesttracker` (`start`);
CREATE INDEX `stacktach_requesttracker_8eb45f9b` ON `stacktach_requesttracker` (`duration`);
CREATE INDEX `stacktach_requesttracker_e490d511` ON `stacktach_requesttracker` (`completed`);
COMMIT;

2
migrations/003_delta.sql Normal file
View File

@ -0,0 +1,2 @@
ALTER TABLE stacktach_rawdata ADD image_type integer;
CREATE INDEX `stacktach_rawdata_cfde77eb` ON `stacktach_rawdata` (`image_type`);

63
migrations/003_image_type.sql Normal file
View File

@ -0,0 +1,63 @@
BEGIN;
CREATE TABLE `stacktach_deployment` (
`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
`name` varchar(50) NOT NULL
)
;
CREATE TABLE `stacktach_rawdata` (
`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
`deployment_id` integer NOT NULL,
`tenant` varchar(50),
`json` longtext NOT NULL,
`routing_key` varchar(50),
`state` varchar(20),
`old_state` varchar(20),
`old_task` varchar(30),
`task` varchar(30),
`image_type` integer,
`when` numeric(20, 6) NOT NULL,
`publisher` varchar(100),
`event` varchar(50),
`service` varchar(50),
`host` varchar(100),
`instance` varchar(50),
`request_id` varchar(50)
)
;
ALTER TABLE `stacktach_rawdata` ADD CONSTRAINT `deployment_id_refs_id_362370d` FOREIGN KEY (`deployment_id`) REFERENCES `stacktach_deployment` (`id`);
CREATE TABLE `stacktach_lifecycle` (
`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
`instance` varchar(50),
`last_state` varchar(50),
`last_task_state` varchar(50),
`last_raw_id` integer
)
;
ALTER TABLE `stacktach_lifecycle` ADD CONSTRAINT `last_raw_id_refs_id_d5fb17d3` FOREIGN KEY (`last_raw_id`) REFERENCES `stacktach_rawdata` (`id`);
CREATE TABLE `stacktach_timing` (
`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
`name` varchar(50) NOT NULL,
`lifecycle_id` integer NOT NULL,
`start_raw_id` integer,
`end_raw_id` integer,
`start_when` numeric(20, 6),
`end_when` numeric(20, 6),
`diff` numeric(20, 6)
)
;
ALTER TABLE `stacktach_timing` ADD CONSTRAINT `lifecycle_id_refs_id_4255ead8` FOREIGN KEY (`lifecycle_id`) REFERENCES `stacktach_lifecycle` (`id`);
ALTER TABLE `stacktach_timing` ADD CONSTRAINT `start_raw_id_refs_id_c32dfe04` FOREIGN KEY (`start_raw_id`) REFERENCES `stacktach_rawdata` (`id`);
ALTER TABLE `stacktach_timing` ADD CONSTRAINT `end_raw_id_refs_id_c32dfe04` FOREIGN KEY (`end_raw_id`) REFERENCES `stacktach_rawdata` (`id`);
CREATE TABLE `stacktach_requesttracker` (
`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
`request_id` varchar(50) NOT NULL,
`lifecycle_id` integer NOT NULL,
`last_timing_id` integer,
`start` numeric(20, 6) NOT NULL,
`duration` numeric(20, 6) NOT NULL,
`completed` bool NOT NULL
)
;
ALTER TABLE `stacktach_requesttracker` ADD CONSTRAINT `lifecycle_id_refs_id_e457729` FOREIGN KEY (`lifecycle_id`) REFERENCES `stacktach_lifecycle` (`id`);
ALTER TABLE `stacktach_requesttracker` ADD CONSTRAINT `last_timing_id_refs_id_f0827cca` FOREIGN KEY (`last_timing_id`) REFERENCES `stacktach_timing` (`id`);
COMMIT;

32
migrations/003_image_type_indexes.sql Normal file
View File

@ -0,0 +1,32 @@
BEGIN;
CREATE INDEX `stacktach_rawdata_4ac6801` ON `stacktach_rawdata` (`deployment_id`);
CREATE INDEX `stacktach_rawdata_2207f86d` ON `stacktach_rawdata` (`tenant`);
CREATE INDEX `stacktach_rawdata_2192f43a` ON `stacktach_rawdata` (`routing_key`);
CREATE INDEX `stacktach_rawdata_355bfc27` ON `stacktach_rawdata` (`state`);
CREATE INDEX `stacktach_rawdata_b716e0bb` ON `stacktach_rawdata` (`old_state`);
CREATE INDEX `stacktach_rawdata_8182be12` ON `stacktach_rawdata` (`old_task`);
CREATE INDEX `stacktach_rawdata_1c149b74` ON `stacktach_rawdata` (`task`);
CREATE INDEX `stacktach_rawdata_cfde77eb` ON `stacktach_rawdata` (`image_type`);
CREATE INDEX `stacktach_rawdata_feaed089` ON `stacktach_rawdata` (`when`);
CREATE INDEX `stacktach_rawdata_878a2906` ON `stacktach_rawdata` (`publisher`);
CREATE INDEX `stacktach_rawdata_a90f9116` ON `stacktach_rawdata` (`event`);
CREATE INDEX `stacktach_rawdata_52c5ef6b` ON `stacktach_rawdata` (`service`);
CREATE INDEX `stacktach_rawdata_38dbea87` ON `stacktach_rawdata` (`host`);
CREATE INDEX `stacktach_rawdata_888b756a` ON `stacktach_rawdata` (`instance`);
CREATE INDEX `stacktach_rawdata_792812e8` ON `stacktach_rawdata` (`request_id`);
CREATE INDEX `stacktach_lifecycle_888b756a` ON `stacktach_lifecycle` (`instance`);
CREATE INDEX `stacktach_lifecycle_9b2555fd` ON `stacktach_lifecycle` (`last_state`);
CREATE INDEX `stacktach_lifecycle_67421a0e` ON `stacktach_lifecycle` (`last_task_state`);
CREATE INDEX `stacktach_lifecycle_dcf9e5f3` ON `stacktach_lifecycle` (`last_raw_id`);
CREATE INDEX `stacktach_timing_52094d6e` ON `stacktach_timing` (`name`);
CREATE INDEX `stacktach_timing_9f222e6b` ON `stacktach_timing` (`lifecycle_id`);
CREATE INDEX `stacktach_timing_efab905a` ON `stacktach_timing` (`start_raw_id`);
CREATE INDEX `stacktach_timing_c8bb8daf` ON `stacktach_timing` (`end_raw_id`);
CREATE INDEX `stacktach_timing_4401d15e` ON `stacktach_timing` (`diff`);
CREATE INDEX `stacktach_requesttracker_792812e8` ON `stacktach_requesttracker` (`request_id`);
CREATE INDEX `stacktach_requesttracker_9f222e6b` ON `stacktach_requesttracker` (`lifecycle_id`);
CREATE INDEX `stacktach_requesttracker_ce616a96` ON `stacktach_requesttracker` (`last_timing_id`);
CREATE INDEX `stacktach_requesttracker_29f4f2ea` ON `stacktach_requesttracker` (`start`);
CREATE INDEX `stacktach_requesttracker_8eb45f9b` ON `stacktach_requesttracker` (`duration`);
CREATE INDEX `stacktach_requesttracker_e490d511` ON `stacktach_requesttracker` (`completed`);
COMMIT;

63
migrations/003_populate_task_and_image.py Normal file
View File

@ -0,0 +1,63 @@
import datetime
import json
import sys
sys.path.append("/stacktach")
from stacktach import datetime_to_decimal as dt
from stacktach import image_type
from stacktach import models
if __name__ != '__main__':
sys.exit(1)
states = {}
def fix_chunk(hours, length):
now = datetime.datetime.utcnow()
start = now - datetime.timedelta(hours=hours+length)
end = now - datetime.timedelta(hours=hours)
dstart = dt.dt_to_decimal(start)
dend = dt.dt_to_decimal(end)
done = 0
updated = 0
block = 0
print "Hours ago (%d to %d) %d - %d" % (hours + length, hours, dstart, dend)
updates = models.RawData.objects.filter(event='compute.instance.update',
when__gt=dstart, when__lte=dend)\
.only('task', 'image_type', 'json')
for raw in updates:
queue, body = json.loads(raw.json)
payload = body.get('payload', {})
task = payload.get('new_task_state', None)
if task != None and task != 'None':
states[task] = states.get(task, 0) + 1
raw.task = task
raw.image_type = image_type.get_numeric_code(payload, raw.image_type)
updated += 1
raw.save()
done += 1
if done >= 10000:
block += 1
done = 0
print "# 10k blocks processed: %d (events %d)" % \
(block, updated)
updated = 0
for kv in states.iteritems():
print "%s = %d" % kv
for day in xrange(0, 90):
hours = day * 24
steps = 12
chunk = 24 / steps
for x in xrange(steps):
fix_chunk(hours, chunk)
hours += chunk

166
reports/pretty.py Normal file
View File

@ -0,0 +1,166 @@
import datetime
import json
import sys
import time
import prettytable
sys.path.append("/stacktach")
from stacktach import datetime_to_decimal as dt
from stacktach import image_type
from stacktach import models
if __name__ != '__main__':
sys.exit(1)
yesterday = datetime.datetime.utcnow().date() - datetime.timedelta(days=1)
if len(sys.argv) == 2:
try:
t = time.strptime(sys.argv[1], "%Y-%m-%d")
yesterday = datetime.datetime(*t[:6])
except Exception, e:
print e
print "Usage: python requests.py YYYY-MM-DD (the end date)"
sys.exit(1)
percentile = 90
hours = 24
start = datetime.datetime(year=yesterday.year, month=yesterday.month,
day=yesterday.day)
end = start + datetime.timedelta(hours=hours-1, minutes=59, seconds=59)
print "Generating report for %s to %s" % (start, end)
dstart = dt.dt_to_decimal(start)
dend = dt.dt_to_decimal(end)
codes = {}
# Get all the instances that have changed in the last N hours ...
updates = models.RawData.objects.filter(event='compute.instance.update',
when__gt=dstart, when__lte=dend)\
.values('instance').distinct()
expiry = 60 * 60 # 1 hour
cmds = ['create', 'rebuild', 'rescue', 'resize', 'snapshot']
failures = {}
durations = {}
attempts = {}
for uuid_dict in updates:
uuid = uuid_dict['instance']
# All the unique Request ID's for this instance during that timespan.
reqs = models.RawData.objects.filter(instance=uuid,
when__gt=dstart, when__lte=dend) \
.values('request_id').distinct()
for req_dict in reqs:
report = False
req = req_dict['request_id']
raws = models.RawData.objects.filter(request_id=req)\
.exclude(event='compute.instance.exists')\
.order_by('when')
start = None
err = None
operation = "aux"
image_type_num = 0
for raw in raws:
if not start:
start = raw.when
if 'error' in raw.routing_key:
err = raw
report = True
for cmd in cmds:
if cmd in raw.event:
operation = cmd
break
if raw.image_type:
image_type_num |= raw.image_type
image = "?"
if image_type.isset(image_type_num, image_type.BASE_IMAGE):
image = "base"
if image_type.isset(image_type_num, image_type.SNAPSHOT_IMAGE):
image = "snap"
if not start:
continue
end = raw.when
diff = end - start
if diff > 3600:
report = True
key = (operation, image)
# Track durations for all attempts, good and bad ...
_durations = durations.get(key, [])
_durations.append(diff)
durations[key] = _durations
attempts[key] = attempts.get(key, 0) + 1
if report:
failures[key] = failures.get(key, 0) + 1
# Print the results ...
cols = ["Operation", "Image", "Min*", "Max*", "Avg*",
"Requests", "# Fail", "Fail %"]
p = prettytable.PrettyTable(cols)
for c in cols[2:]:
p.align[c] = 'r'
p.sortby = cols[0]
pct = (float(100 - percentile) / 2.0) / 100.0
print "* Using %d-th percentile for results (+/-%.1f%% cut)" % \
(percentile, pct * 100.0)
total = 0
failure_total = 0
for key, count in attempts.iteritems():
total += count
operation, image = key
failure_count = failures.get(key, 0)
failure_total += failure_count
failure_percentage = float(failure_count) / float(count)
_failure_percentage = "%.1f%%" % (failure_percentage * 100.0)
# N-th % of durations ...
_values = durations[key]
_values.sort()
_outliers = int(float(len(_values)) * pct)
if _outliers > 0:
before = len(_values)
_values = _values[_outliers:-_outliers]
print "culling %d -> %d" % (before, len(_values))
_min = 99999999
_max = 0
_total = 0.0
for value in _values:
_min = min(_min, value)
_max = max(_max, value)
_total += float(value)
_avg = float(_total) / float(len(_values))
_fmin = dt.sec_to_str(_min)
_fmax = dt.sec_to_str(_max)
_favg = dt.sec_to_str(_avg)
p.add_row([operation, image, _fmin, _fmax, _favg, count,
failure_count, _failure_percentage])
print p
print "Total: %d, Failures: %d, Failure Rate: %.1f%%" % \
(total, failure_total,
(float(failure_total)/float(total)) * 100.0)

258
reports/requests.py Normal file
View File

@ -0,0 +1,258 @@
import datetime
import json
import sys
import time
import prettytable
sys.path.append("/stacktach")
from stacktach import datetime_to_decimal as dt
from stacktach import image_type
from stacktach import models
if __name__ != '__main__':
sys.exit(1)
yesterday = datetime.datetime.utcnow().date() - datetime.timedelta(days=1)
if len(sys.argv) == 2:
try:
t = time.strptime(sys.argv[1], "%Y-%m-%d")
yesterday = datetime.datetime(*t[:6])
except Exception, e:
print e
print "Usage: python requests.py YYYY-MM-DD (the end date)"
sys.exit(1)
hours = 0
length = 6
start = datetime.datetime(year=yesterday.year, month=yesterday.month,
day=yesterday.day)
end = start + datetime.timedelta(hours=length-1, minutes=59, seconds=59)
print "Generating report for %s to %s" % (start, end)
dstart = dt.dt_to_decimal(start)
dend = dt.dt_to_decimal(end)
codes = {}
# Get all the instances that have changed in the last N hours ...
updates = models.RawData.objects.filter(event='compute.instance.update',
when__gt=dstart, when__lte=dend)\
.values('instance').distinct()
expiry = 60 * 60 # 1 hour
cmds = ['create', 'rebuild', 'rescue', 'resize', 'snapshot']
failures = {}
causes = {}
durations = {}
error_messages = {}
successes = {}
tenant_issues = {}
for uuid_dict in updates:
uuid = uuid_dict['instance']
# All the unique Request ID's for this instance during that timespan.
reqs = models.RawData.objects.filter(instance=uuid,
when__gt=dstart, when__lte=dend) \
.values('request_id').distinct()
for req_dict in reqs:
report = False
req = req_dict['request_id']
raws = models.RawData.objects.filter(request_id=req)\
.exclude(event='compute.instance.exists')\
.order_by('when')
start = None
err = None
operation = "aux"
platform = 0
tenant = 0
cell = "unk"
for raw in raws:
if not start:
start = raw.when
if 'error' in raw.routing_key:
err = raw
report = True
if raw.tenant:
if tenant > 0 and raw.tenant != tenant:
print "Conflicting tenant ID", raw.tenant, tenant
tenant = raw.tenant
for cmd in cmds:
if cmd in raw.event:
operation = cmd
cell = raw.deployment.name
break
if raw.image_type > 0:
platform = raw.image_type
if not start:
continue
end = raw.when
diff = end - start
if diff > 3600:
report = True
key = (operation, platform, cell)
# Track durations for all attempts, good and bad ...
duration_min, duration_max, duration_count, duration_total = \
durations.get(key, (9999999, 0, 0, 0))
duration_min = min(duration_min, diff)
duration_max = max(duration_max, diff)
duration_count += 1
duration_total += diff
durations[key] = (duration_min, duration_max, duration_count,
duration_total)
if not report:
successes[key] = successes.get(key, 0) + 1
else:
print "------", uuid, "----------"
print " Req:", req
print " Duration: %.2f minutes" % (diff / 60)
print " Operation:", operation
print " Platform:", image_type.readable(platform)
cause = "> %d min" % (expiry / 60)
failures[key] = failures.get(key, 0) + 1
tenant_issues[tenant] = tenant_issues.get(tenant, 0) + 1
if err:
queue, body = json.loads(err.json)
payload = body['payload']
print "Error. EventID: %s, Tenant %s, Service %s, Host %s, "\
"Deployment %s, Event %s, When %s"\
% (err.id, err.tenant, err.service, err.host,
err.deployment.name,
err.event, dt.dt_from_decimal(err.when))
exc = payload.get('exception')
if exc:
# group the messages ...
exc_str = str(exc)
print exc_str
error_messages[exc_str] = \
error_messages.get(exc_str, 0) + 1
# extract the code, if any ...
code = exc.get('kwargs', {}).get('code')
if code:
codes[code] = codes.get(code, 0) + 1
cause = code
cause_key = (key, cause)
causes[cause_key] = causes.get(cause_key, 0) + 1
def dump_breakdown(totals, label):
p = prettytable.PrettyTable(["Category", "Count"])
for k, v in totals.iteritems():
p.add_row([k, v])
print label
p.sortby = 'Count'
print p
def dump_summary(info, label):
print "-- %s by operation by cell by platform --" % (label,)
p = prettytable.PrettyTable(["Operation", "Cell", "Platform", "Count",
"Min", "Max", "Avg"])
for c in ["Count", "Min", "Max", "Avg"]:
p.align[c] = 'r'
total = 0
op_totals = {}
cell_totals = {}
platform_totals = {}
for key, count in info.iteritems():
operation, platform, cell = key
readable = image_type.readable(platform)
text = "n/a"
if readable:
text = ", ".join(readable)
_min, _max, _count, _total = durations[key]
_avg = float(_total) / float(_count)
_fmin = dt.sec_to_str(_min)
_fmax = dt.sec_to_str(_max)
_favg = dt.sec_to_str(_avg * 100.0)
op_totals[operation] = op_totals.get(operation, 0) + count
cell_totals[cell] = cell_totals.get(cell, 0) + count
platform_totals[text] = platform_totals.get(text, 0) + count
p.add_row([operation, cell, text, count, _fmin, _fmax, _favg])
total += count
p.sortby = 'Count'
print p
dump_breakdown(op_totals, "Total %s by Operation" % label)
dump_breakdown(cell_totals, "Total %s by Cell" % label)
dump_breakdown(platform_totals, "Total %s by Platform" % label)
print
return total
print
print "SUMMARY"
print
good = dump_summary(successes, "Success")
bad = dump_summary(failures, "Failures")
print "====================================================="
print "Total Success: %d Total Failure: %d" % (good, bad)
print
print "-- Errors by Tenant --"
p = prettytable.PrettyTable(["Tenant", "Count"])
for tenant, count in tenant_issues.iteritems():
p.add_row([tenant, count])
p.sortby = 'Count'
print p
print
print "-- Return code counts --"
p = prettytable.PrettyTable(["Return Code", "Count"])
for k, v in codes.iteritems():
p.add_row([k, v])
p.sortby = 'Count'
print p
print
print "-- Cause breakdown --"
p = prettytable.PrettyTable(["Cause", "Operation", "Cell", "Platform", "Count"])
for cause_key, count in causes.iteritems():
key, cause = cause_key
operation, platform, cell = key
readable = image_type.readable(platform)
text = "n/a"
if readable:
text = ", ".join(readable)
p.add_row([cause, operation, cell, text, count])
p.sortby = 'Count'
print p
print
print "-- Error Message Counts --"
p = prettytable.PrettyTable(["Count", "Message"])
for k, v in error_messages.iteritems():
p.add_row([v, k[:80]])
p.sortby = 'Count'
print p

15
stacktach/datetime_to_decimal.py
View File

@ -12,8 +12,23 @@ def dt_to_decimal(utc):
def dt_from_decimal(dec):
if dec == None:
return "n/a"
integer = int(dec)
micro = (dec - decimal.Decimal(integer)) * decimal.Decimal(1000000)
daittyme = datetime.datetime.utcfromtimestamp(integer)
return daittyme.replace(microsecond=micro)
def sec_to_str(sec):
sec = int(sec)
if sec < 60:
return "%ds" % sec
minutes = sec / 60
sec = sec % 60
if minutes < 60:
return "%d:%02ds" % (minutes, sec)
hours = minutes / 60
minutes = minutes % 60
return "%02d:%02d:%02d" % (hours, minutes, sec)

60
stacktach/image_type.py Normal file
View File

@ -0,0 +1,60 @@
BASE_IMAGE = 0x1
SNAPSHOT_IMAGE = 0x2
LINUX_IMAGE = 0x10
OS_UBUNTU = 0x100
OS_DEBIAN = 0x200
OS_CENTOS = 0x400
OS_RHEL = 0x800
def isset(num, flag):
if not num:
return False
return num & flag > 0
flags = {'base' : BASE_IMAGE,
'snapshot' : SNAPSHOT_IMAGE,
'linux' : LINUX_IMAGE,
'ubuntu' : OS_UBUNTU,
'debian' : OS_DEBIAN,
'centos' : OS_CENTOS,
'rhel' : OS_RHEL}
def readable(num):
result = []
for k, v in flags.iteritems():
if isset(num, v):
result.append(k)
return result
def get_numeric_code(payload, default=0):
meta = payload.get('image_meta', {})
if default == None:
default = 0
num = default
image_type = meta.get('image_type', '')
if image_type == 'base':
num |= BASE_IMAGE
if image_type == 'snapshot':
num |= SNAPSHOT_IMAGE
os_type = meta.get('os_type', payload.get('os_type', ''))
if os_type == 'linux':
num |= LINUX_IMAGE
os_distro = meta.get('os_distro', '')
if os_distro == 'ubuntu':
num |= OS_UBUNTU
if os_distro == 'debian':
num |= OS_DEBIAN
if os_distro == 'centos':
num |= OS_CENTOS
if os_distro == 'rhel':
num |= OS_RHEL
return num

3
stacktach/models.py
View File

@ -34,6 +34,9 @@ class RawData(models.Model):
blank=True, db_index=True)
old_task = models.CharField(max_length=30, null=True,
blank=True, db_index=True)
task = models.CharField(max_length=30, null=True,
blank=True, db_index=True)
image_type = models.IntegerField(null=True, default=0, db_index=True)
when = models.DecimalField(max_digits=20, decimal_places=6,
db_index=True)
publisher = models.CharField(max_length=100, null=True,

22
stacktach/views.py
View File

@ -1,18 +1,18 @@
# Copyright 2012 - Dark Secret Software Inc.
from django import db
from django import http
from django.shortcuts import render_to_response
from django import template
from stacktach import db as stackdb
from stacktach import models
from stacktach import datetime_to_decimal as dt
import datetime
import json
import pprint
from django import http
from django.shortcuts import render_to_response
from django import template
from stacktach import datetime_to_decimal as dt
from stacktach import db as stackdb
from stacktach import models
from stacktach import image_type
STACKDB = stackdb
@ -21,7 +21,9 @@ def _extract_states(payload):
return {
'state' : payload.get('state', ""),
'old_state' : payload.get('old_state', ""),
'old_task' : payload.get('old_task_state', "")
'old_task' : payload.get('old_task_state', ""),
'task' : payload.get('new_task_state', ""),
'image_type' : image_type.get_numeric_code(payload)
}

33
worker/worker.py
View File

@ -22,6 +22,7 @@ import kombu
import kombu.entity
import kombu.mixins
import logging
import sys
import time
from pympler.process import ProcessMemoryInfo
@ -33,6 +34,8 @@ LOG = logging.getLogger(__name__)
LOG.setLevel(logging.DEBUG)
handler = logging.handlers.TimedRotatingFileHandler('worker.log',
when='h', interval=6, backupCount=4)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
LOG.addHandler(handler)
@ -144,16 +147,20 @@ def run(deployment_config):
transport="librabbitmq",
virtual_host=virtual_host)
while continue_running():
LOG.debug("Processing on '%s'" % name)
with kombu.connection.BrokerConnection(**params) as conn:
try:
consumer = NovaConsumer(name, conn, deployment, durable,
queue_arguments)
consumer.run()
except Exception as e:
LOG.exception("name=%s, exception=%s. Reconnecting in 5s" %
(name, e))
time.sleep(5)
LOG.debug("Completed processing on '%s'" % name)
while True:
try:
LOG.debug("Processing on '%s'" % name)
with kombu.connection.BrokerConnection(**params) as conn:
try:
consumer = NovaConsumer(name, conn, deployment, durable)
consumer.run()
except Exception as e:
LOG.exception("name=%s, exception=%s. Reconnecting in 5s" %
(name, e))
time.sleep(5)
LOG.debug("Completed processing on '%s'" % name)
except:
e = sys.exc_info()[0]
msg = "Uncaught exception: deployment=%s, exception=%s. Retrying in 5s"
LOG.exception(msg % (name, e))
time.sleep(5)