From 03c5a51527f8d5ceef6287e5ca2cfd0f3c4f4092 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 19 Nov 2012 11:14:52 -0400 Subject: [PATCH 1/7] fixed up a bunch of little gotchas. kpi working --- settings.py | 2 +- stacktach/models.py | 5 +++-- stacktach/stacky_server.py | 1 + stacktach/views.py | 7 ++++--- worker/stacktach.sh | 2 +- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/settings.py b/settings.py index 05068b6..a972fb5 100644 --- a/settings.py +++ b/settings.py @@ -16,7 +16,7 @@ except ImportError: db_password = os.environ['STACKTACH_DB_PASSWORD'] install_dir = os.environ['STACKTACH_INSTALL_DIR'] -DEBUG = False +DEBUG = True TEMPLATE_DEBUG = DEBUG ADMINS = ( diff --git a/stacktach/models.py b/stacktach/models.py index f09e3ff..2a5310b 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -81,7 +81,8 @@ class Timing(models.Model): start_raw = models.ForeignKey(RawData, related_name='+', null=True) end_raw = models.ForeignKey(RawData, related_name='+', null=True) - start_when = models.DecimalField(null=True, max_digits=20, decimal_places=6) + start_when = models.DecimalField(null=True, max_digits=20, + decimal_places=6) end_when = models.DecimalField(null=True, max_digits=20, decimal_places=6) diff = models.DecimalField(null=True, max_digits=20, decimal_places=6) @@ -93,7 +94,7 @@ class RequestTracker(models.Model): final .end event (with the same Request ID).""" request_id = models.CharField(max_length=50, db_index=True) lifecycle = models.ForeignKey(Lifecycle) - last_timing = models.ForeignKey(Timing, null=true) + last_timing = models.ForeignKey(Timing, null=True) start = models.DecimalField(max_digits=20, decimal_places=6) duration = models.DecimalField(max_digits=20, decimal_places=6) diff --git a/stacktach/stacky_server.py b/stacktach/stacky_server.py index 127d400..4daeaba 100644 --- a/stacktach/stacky_server.py +++ b/stacktach/stacky_server.py @@ -281,6 +281,7 @@ def do_watch(request, deployment_id): def do_kpi(request): yesterday = datetime.datetime.utcnow() - datetime.timedelta(days=1) + yesterday = dt.dt_to_decimal(yesterday) trackers = models.RequestTracker.objects.select_related() \ .exclude(last_timing=None) \ .exclude(start__lt=yesterday) \ diff --git a/stacktach/views.py b/stacktach/views.py index 042ae0b..f71cecc 100644 --- a/stacktach/views.py +++ b/stacktach/views.py @@ -87,7 +87,7 @@ def start_kpi_tracking(lifecycle, raw): start=raw.when, lifecycle=lifecycle, last_timing=None, - duration=0.0) + duration=str(0.0)) tracker.save() @@ -102,7 +102,8 @@ def update_kpi(lifecycle, timing, raw): Until then, we'll take the lazy route and be aware of these potential fence-post issues.""" - trackers = models.RequestTracker.objects.filter(request_id=raw.request.id) + trackers = models.RequestTracker.objects.\ + filter(request_id=raw.request_id) if len(trackers) == 0: return @@ -146,7 +147,7 @@ def aggregate(raw): if not step in ['start', 'end']: # Perhaps it's an operation initiated in the API? - start_kpi_tracking(lifecyle, raw) + start_kpi_tracking(lifecycle, raw) return # We are going to try to track every event pair that comes diff --git a/worker/stacktach.sh b/worker/stacktach.sh index be51937..e048438 100755 --- a/worker/stacktach.sh +++ b/worker/stacktach.sh @@ -1,6 +1,6 @@ #!/bin/bash -WORKDIR=/srv/www/stacktach/django/stproject/ +WORKDIR=/srv/www/stacktach/app DAEMON=/usr/bin/python ARGS=$WORKDIR/worker/start_workers.py PIDFILE=/var/run/stacktach.pid From 4f36d540d011a3a90eaf6e2f95c3e4457df02f11 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 19 Nov 2012 11:53:51 -0400 Subject: [PATCH 2/7] whoops, turned debug back on --- settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.py b/settings.py index a972fb5..05068b6 100644 --- a/settings.py +++ b/settings.py @@ -16,7 +16,7 @@ except ImportError: db_password = os.environ['STACKTACH_DB_PASSWORD'] install_dir = os.environ['STACKTACH_INSTALL_DIR'] -DEBUG = True +DEBUG = False TEMPLATE_DEBUG = DEBUG ADMINS = ( From 38c591ebaad0c45a66e2c96dc8c1b110828d2c2c Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Tue, 27 Nov 2012 14:57:15 -0400 Subject: [PATCH 3/7] kpi command now takes optional tenant id. Fixes rounding error. --- settings.py | 2 +- stacktach/models.py | 2 +- stacktach/stacky_server.py | 7 ++++--- stacktach/urls.py | 1 + 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/settings.py b/settings.py index 05068b6..a972fb5 100644 --- a/settings.py +++ b/settings.py @@ -16,7 +16,7 @@ except ImportError: db_password = os.environ['STACKTACH_DB_PASSWORD'] install_dir = os.environ['STACKTACH_INSTALL_DIR'] -DEBUG = False +DEBUG = True TEMPLATE_DEBUG = DEBUG ADMINS = ( diff --git a/stacktach/models.py b/stacktach/models.py index 2a5310b..b50e6db 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -18,7 +18,7 @@ from django.db import models class Deployment(models.Model): - name = models.CharField(max_length=50) + name = models.CharField(max_length=50, db_index=True) def get_or_create_deployment(name): diff --git a/stacktach/stacky_server.py b/stacktach/stacky_server.py index 4daeaba..327bb25 100644 --- a/stacktach/stacky_server.py +++ b/stacktach/stacky_server.py @@ -67,7 +67,7 @@ def sec_to_time(diff): seconds -= (hours * SECS_PER_HOUR) minutes = seconds / 60 seconds -= (minutes * 60) - usec = ('%.2f' % usec).lstrip('0') + usec = str(usec)[1:4] return "%dd %02d:%02d:%02d%s" % (days, hours, minutes, seconds, usec) @@ -279,7 +279,7 @@ def do_watch(request, deployment_id): return rsp([c, results, str(dec_now)]) -def do_kpi(request): +def do_kpi(request, tenant_id=None): yesterday = datetime.datetime.utcnow() - datetime.timedelta(days=1) yesterday = dt.dt_to_decimal(yesterday) trackers = models.RequestTracker.objects.select_related() \ @@ -293,6 +293,7 @@ def do_kpi(request): end_event = track.last_timing.end_raw event = end_event.event[:-len(".end")] uuid = track.lifecycle.instance - results.append([event, sec_to_time(track.duration), + if tenant_id == None or (tenant_id == end_event.tenant): + results.append([event, sec_to_time(track.duration), uuid, end_event.deployment.name]) return rsp(results) diff --git a/stacktach/urls.py b/stacktach/urls.py index 867b9ae..212cdea 100644 --- a/stacktach/urls.py +++ b/stacktach/urls.py @@ -17,6 +17,7 @@ urlpatterns = patterns('', url(r'stacky/watch/(?P\d+)/$', 'stacktach.stacky_server.do_watch'), url(r'stacky/kpi/$', 'stacktach.stacky_server.do_kpi'), + url(r'stacky/kpi/(?P\d+)/$', 'stacktach.stacky_server.do_kpi'), url(r'^(?P\d+)/$', 'stacktach.views.home', name='home'), url(r'^(?P\d+)/details/(?P\w+)/(?P\d+)/$', From d53c77c03b4f559a7a05d454a248144bb2f525b2 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Tue, 27 Nov 2012 14:57:15 -0400 Subject: [PATCH 4/7] kpi command now takes optional tenant id. Fixes rounding error. --- stacktach/stacky_server.py | 7 ++++--- stacktach/urls.py | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/stacktach/stacky_server.py b/stacktach/stacky_server.py index 4daeaba..327bb25 100644 --- a/stacktach/stacky_server.py +++ b/stacktach/stacky_server.py @@ -67,7 +67,7 @@ def sec_to_time(diff): seconds -= (hours * SECS_PER_HOUR) minutes = seconds / 60 seconds -= (minutes * 60) - usec = ('%.2f' % usec).lstrip('0') + usec = str(usec)[1:4] return "%dd %02d:%02d:%02d%s" % (days, hours, minutes, seconds, usec) @@ -279,7 +279,7 @@ def do_watch(request, deployment_id): return rsp([c, results, str(dec_now)]) -def do_kpi(request): +def do_kpi(request, tenant_id=None): yesterday = datetime.datetime.utcnow() - datetime.timedelta(days=1) yesterday = dt.dt_to_decimal(yesterday) trackers = models.RequestTracker.objects.select_related() \ @@ -293,6 +293,7 @@ def do_kpi(request): end_event = track.last_timing.end_raw event = end_event.event[:-len(".end")] uuid = track.lifecycle.instance - results.append([event, sec_to_time(track.duration), + if tenant_id == None or (tenant_id == end_event.tenant): + results.append([event, sec_to_time(track.duration), uuid, end_event.deployment.name]) return rsp(results) diff --git a/stacktach/urls.py b/stacktach/urls.py index 867b9ae..212cdea 100644 --- a/stacktach/urls.py +++ b/stacktach/urls.py @@ -17,6 +17,7 @@ urlpatterns = patterns('', url(r'stacky/watch/(?P\d+)/$', 'stacktach.stacky_server.do_watch'), url(r'stacky/kpi/$', 'stacktach.stacky_server.do_kpi'), + url(r'stacky/kpi/(?P\d+)/$', 'stacktach.stacky_server.do_kpi'), url(r'^(?P\d+)/$', 'stacktach.views.home', name='home'), url(r'^(?P\d+)/details/(?P\w+)/(?P\d+)/$', From 2fb3a5b482145bedf6ece48d938f90336c691367 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 5 Dec 2012 16:25:09 -0600 Subject: [PATCH 5/7] missing when index. Where clause on latest_raw query --- stacktach/models.py | 3 ++- stacktach/views.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/stacktach/models.py b/stacktach/models.py index 2a5310b..4852038 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -38,7 +38,8 @@ class RawData(models.Model): blank=True, db_index=True) old_task = models.CharField(max_length=30, null=True, blank=True, db_index=True) - when = models.DecimalField(max_digits=20, decimal_places=6) + when = models.DecimalField(max_digits=20, decimal_places=6, + db_index=True) publisher = models.CharField(max_length=100, null=True, blank=True, db_index=True) event = models.CharField(max_length=50, null=True, diff --git a/stacktach/views.py b/stacktach/views.py index f71cecc..c01a922 100644 --- a/stacktach/views.py +++ b/stacktach/views.py @@ -314,7 +314,9 @@ def latest_raw(request, deployment_id): """This is the 2sec ticker that updates the Recent Activity box.""" deployment_id = int(deployment_id) c = _default_context(request, deployment_id) - query = models.RawData.objects.select_related() + then = datetime.datetime.utcnow() - datetime.timedelta(hours=1) + thend = dt.dt_to_decimal(then) + query = models.RawData.objects.select_related().filter(when__gt=thend) if deployment_id > 0: query = query.filter(deployment=deployment_id) rows = query.order_by('-when')[:20] From 1df8b4a7a664a2f568e1273f78feb66bf906d728 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Tue, 11 Dec 2012 09:14:38 -0400 Subject: [PATCH 6/7] added more missing indicies --- stacktach/models.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/stacktach/models.py b/stacktach/models.py index 4852038..eabc548 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -86,7 +86,8 @@ class Timing(models.Model): decimal_places=6) end_when = models.DecimalField(null=True, max_digits=20, decimal_places=6) - diff = models.DecimalField(null=True, max_digits=20, decimal_places=6) + diff = models.DecimalField(null=True, max_digits=20, decimal_places=6, + db_index=True) class RequestTracker(models.Model): @@ -97,7 +98,8 @@ class RequestTracker(models.Model): lifecycle = models.ForeignKey(Lifecycle) last_timing = models.ForeignKey(Timing, null=True) start = models.DecimalField(max_digits=20, decimal_places=6) - duration = models.DecimalField(max_digits=20, decimal_places=6) + duration = models.DecimalField(max_digits=20, decimal_places=6, + db_index=True) # Not used ... but soon hopefully. - completed = models.BooleanField(default=False) + completed = models.BooleanField(default=False, db_index=True) From 78b45147c7cbb01968129d30b697b16930f7d0e9 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Tue, 11 Dec 2012 09:48:44 -0400 Subject: [PATCH 7/7] missing indices on kpi table --- stacktach/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stacktach/models.py b/stacktach/models.py index eabc548..9fde6b2 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -96,8 +96,8 @@ class RequestTracker(models.Model): final .end event (with the same Request ID).""" request_id = models.CharField(max_length=50, db_index=True) lifecycle = models.ForeignKey(Lifecycle) - last_timing = models.ForeignKey(Timing, null=True) - start = models.DecimalField(max_digits=20, decimal_places=6) + last_timing = models.ForeignKey(Timing, null=True, db_index=True) + start = models.DecimalField(max_digits=20, decimal_places=6, db_index=True) duration = models.DecimalField(max_digits=20, decimal_places=6, db_index=True)