Stored report support.

The "pretty" report can now be stored in the stacktach database
and retrieved via REST or stacky.

Also better command line support for the "pretty" report.

migrations/005_reports.txt

@@ -0,0 +1,3 @@
+do 
+python manage.py syncdb 
+to add the JsonReport table

reports/pretty.py

@@ -7,23 +7,24 @@ import time
 import prettytable
 
 sys.path.append("/stacktach")
-sys.path.append(".")
 
 from stacktach import datetime_to_decimal as dt
 from stacktach import image_type
 from stacktach import models
 
 
-def make_report(yesterday=None, start_hour=0, hours=24, percentile=90, store=False):
+def make_report(yesterday=None, start_hour=0, hours=24, percentile=90,
+                store=False):
     if not yesterday:
-        yesterday = datetime.datetime.utcnow().date() - datetime.timedelta(days=1)
+        yesterday = datetime.datetime.utcnow().date() - \
+                    datetime.timedelta(days=1)
 
-    start = datetime.datetime(year=yesterday.year, month=yesterday.month, 
+    rstart = datetime.datetime(year=yesterday.year, month=yesterday.month,
-                              day=yesterday.day, hour=start_hour) 
+                              day=yesterday.day, hour=start_hour)
-    end = start + datetime.timedelta(hours=hours-1, minutes=59, seconds=59)
+    rend = rstart + datetime.timedelta(hours=hours-1, minutes=59, seconds=59)
 
-    dstart = dt.dt_to_decimal(start)
+    dstart = dt.dt_to_decimal(rstart)
-    dend = dt.dt_to_decimal(end)
+    dend = dt.dt_to_decimal(rend)
 
     codes = {}
 
@@ -52,8 +53,8 @@ def make_report(yesterday=None, start_hour=0, hours=24, percentile=90, store=Fal
             report = False
             req = req_dict['request_id']
             raws = models.RawData.objects.filter(request_id=req)\
-                                         .exclude(event='compute.instance.exists')\
+                                      .exclude(event='compute.instance.exists')\
-                                         .order_by('when')
+                                      .order_by('when')
 
             start = None
             err = None
@@ -74,7 +75,7 @@ def make_report(yesterday=None, start_hour=0, hours=24, percentile=90, store=Fal
                         break
 
                 if raw.image_type:
-                    image_type_num |= raw.image_type                 
+                    image_type_num |= raw.image_type
 
             image = "?"
             if image_type.isset(image_type_num, image_type.BASE_IMAGE):
@@ -106,8 +107,8 @@ def make_report(yesterday=None, start_hour=0, hours=24, percentile=90, store=Fal
     # Summarize the results ...
     report = []
     pct = (float(100 - percentile) / 2.0) / 100.0
-    details = {'percentile': percentile, 'pct': pct, 'hours': hours, 
+    details = {'percentile': percentile, 'pct': pct, 'hours': hours,
-                   'start': start, 'end': end}
+                   'start': float(dstart), 'end': float(dend)}
     report.append(details)
 
     cols = ["Operation", "Image", "Min*", "Max*", "Avg*",
@@ -143,13 +144,13 @@ def make_report(yesterday=None, start_hour=0, hours=24, percentile=90, store=Fal
         _fmax = dt.sec_to_str(_max)
         _favg = dt.sec_to_str(_avg)
 
-        report.add_row([operation, image, _fmin, _fmax, _favg, count, 
+        report.append([operation, image, _fmin, _fmax, _favg, count,
-                   failure_count, failure_percentage])
+                       failure_count, failure_percentage])
 
     details['total'] = total
-    details['failures'] = failures
+    details['failure_total'] = failure_total
     details['failure_rate'] = (float(failure_total)/float(total)) * 100.0
-    return report
+    return (rstart, rend, report)
 
 
 def valid_date(date):
@@ -157,39 +158,57 @@ def valid_date(date):
         t = time.strptime(date, "%Y-%m-%d")
         return datetime.datetime(*t[:6])
     except Exception, e:
-        raise argparse.ArgumentTypeError("'%s' is not in YYYY-MM-DD format." % date)
+        raise argparse.ArgumentTypeError(
+                                    "'%s' is not in YYYY-MM-DD format." % date)
 
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser('StackTach Nova Usage Summary Report')
-    parser.add_argument('--utcdate', help='Report start date YYYY-MM-DD. Default yesterday midnight.', 
+    parser.add_argument('--utcdate',
-                                     type=valid_date, default=None)
+            help='Report start date YYYY-MM-DD. Default yesterday midnight.',
-    parser.add_argument('--hours', help='Report span in hours. Default: 24', default=24, type=int)
+            type=valid_date, default=None)
-    parser.add_argument('--start_hour', help='Starting hour 0-23. Default: 0', default=0, type=int)
+    parser.add_argument('--hours',
-    parser.add_argument('--percentile', help='Percentile for timings. Default: 90', default=90, type=int)
+            help='Report span in hours. Default: 24', default=24,
-    parser.add_argument('--store', help='Store report in database. Default: False', default=False, 
+            type=int)
-                                                                action="store_true")
+    parser.add_argument('--start_hour',
-    parser.add_argument('--silent', help="Do not show summary report. Default: False", default=False, 
+            help='Starting hour 0-23. Default: 0', default=0,
-                                                                action="store_true")
+            type=int)
+    parser.add_argument('--percentile',
+            help='Percentile for timings. Default: 90', default=90,
+            type=int)
+    parser.add_argument('--store',
+            help='Store report in database. Default: False',
+            default=False, action="store_true")
+    parser.add_argument('--silent',
+            help="Do not show summary report. Default: False",
+            default=False, action="store_true")
     args = parser.parse_args()
 
     yesterday = args.utcdate
     percentile = args.percentile
     hours = args.hours
     start_hour = args.start_hour
+    store_report = args.store
 
-    print args
+    start, end, raw_report = make_report(yesterday, start_hour, hours,
-    sys.exit(1)
+                                         percentile, store_report)
-    raw_report = make_report(yesterday, start_hour, hours, percentile, args['store'])
+    details = raw_report[0]
+    pct = details['pct']
 
-    if not args.show:
+    if store_report:
+        values = {'json': json.dumps(raw_report),
+                  'created': float(dt.dt_to_decimal(datetime.datetime.utcnow())),
+                  'period_start': start,
+                  'period_end': end,
+                  'version': 1,
+                  'name': 'summary report'}
+        report = models.JsonReport(**values)
+        report.save()
+        print "Report stored (id=%d)" % report.id
+
+    if args.silent:
         sys.exit(1)
 
-    details = raw_report[0]
-    percentile = details['percentile']
-    pct = details['pct']
-    start = details['start']
-    end = details['end']
     print "Report for %s to %s" % (start, end)
 
     cols = raw_report[1]
@@ -203,11 +222,13 @@ if __name__ == '__main__':
     print "* Using %d-th percentile for results (+/-%.1f%% cut)" % \
                                 (percentile, pct * 100.0)
     for row in raw_report[2:]:
-        p.add_row(row)
+        frow = row[:]
+        frow[-1] = "%.1f%%" % (row[-1] * 100.0)
+        p.add_row(frow)
     print p
 
     total = details['total']
     failure_total = details['failure_total']
+    failure_rate = details['failure_rate']
     print "Total: %d, Failures: %d, Failure Rate: %.1f%%" % \
-                    (total, failure_total, 
+                    (total, failure_total, failure_rate)
-                        (float(failure_total)/float(total)) * 100.0)

settings.py

@@ -19,7 +19,7 @@ except ImportError:
     db_password = os.environ['STACKTACH_DB_PASSWORD']
     install_dir = os.environ['STACKTACH_INSTALL_DIR']
 
-DEBUG = True
+DEBUG = False
 TEMPLATE_DEBUG = DEBUG
 
 ADMINS = (

stacktach/models.py

@@ -161,7 +161,7 @@ class JsonReport(models.Model):
        via stacky/rest. All DateTimes are UTC."""
     period_start = models.DateTimeField(db_index=True)
     period_end = models.DateTimeField(db_index=True)
-    created = models.DateTimeField(db_index=True)
+    created = models.DecimalField(max_digits=20, decimal_places=6, db_index=True)
     name = models.CharField(max_length=50, db_index=True)
     version = models.IntegerField(default=1)
     json = models.TextField()

stacktach/stacky_server.py

@@ -4,6 +4,7 @@ import json
 
 from django.db.models import Q
 from django.http import HttpResponse
+from django.shortcuts import get_object_or_404
 
 import datetime_to_decimal as dt
 import models
@@ -375,3 +376,30 @@ def do_list_usage_exists(request):
                         exist.status])
 
     return rsp(results)
+
+
+def do_jsonreports(request):
+    yesterday = datetime.datetime.utcnow() - datetime.timedelta(days=1)
+    now = datetime.datetime.utcnow()
+    yesterday = dt.dt_to_decimal(yesterday)
+    now = dt.dt_to_decimal(now)
+    _from = float(request.GET.get('created_from', yesterday))
+    _to = float(request.GET.get('created_to', now))
+    reports = models.JsonReport.objects.filter(created__gte=_from,
+                                               created__lte=_to)
+    results = []
+    results.append(['Id', 'Start', 'End', 'Created', 'Name', 'Version'])
+    for report in reports:
+        results.append([report.id,
+                        float(dt.dt_to_decimal(report.period_start)),
+                        float(dt.dt_to_decimal(report.period_end)),
+                        float(report.created),
+                        report.name,
+                        report.version])
+    return rsp(results)
+
+
+def do_jsonreport(request, report_id):
+    report_id = int(report_id)
+    report = get_object_or_404(models.JsonReport, pk=report_id)
+    return rsp(report.json)

stacktach/urls.py

@@ -12,6 +12,9 @@ urlpatterns = patterns('',
     url(r'stacky/timings/uuid/$', 'stacktach.stacky_server.do_timings_uuid'),
     url(r'stacky/summary/$', 'stacktach.stacky_server.do_summary'),
     url(r'stacky/request/$', 'stacktach.stacky_server.do_request'),
+    url(r'stacky/reports/$', 'stacktach.stacky_server.do_jsonreports'),
+    url(r'stacky/report/(?P<report_id>\d+)/$',
+                            'stacktach.stacky_server.do_jsonreport'),
     url(r'stacky/show/(?P<event_id>\d+)/$',
                                         'stacktach.stacky_server.do_show'),
     url(r'stacky/watch/(?P<deployment_id>\d+)/$',