started extracting code

README

@@ -0,0 +1,3 @@
+StackTach is a debugging tool for OpenStack Nova.
+
+It takes events from AMQP and sends them to the StackTach server for web display.

manage.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+from django.core.management import execute_manager
+import imp
+try:
+    imp.find_module('settings') # Assumed to be in the same directory.
+except ImportError:
+    import sys
+    sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n" % __file__)
+    sys.exit(1)
+
+import settings
+
+if __name__ == "__main__":
+    execute_manager(settings)

stacktach/models.py

@@ -0,0 +1,53 @@
+# Copyright 2012 - Dark Secret Software Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from django import forms
+from django.db import models
+
+
+class Tenant(models.Model):
+    email = models.CharField(max_length=50)
+    project_name = models.CharField(max_length=50)
+    tenant_id = models.AutoField(primary_key=True, unique=True)
+
+
+class RawData(models.Model):
+    tenant = models.ForeignKey(Tenant, db_index=True,
+                               to_field='tenant_id')
+    nova_tenant = models.CharField(max_length=50, null=True,
+                                   blank=True, db_index=True)
+    json = models.TextField()
+    routing_key = models.CharField(max_length=50, null=True,
+                                   blank=True, db_index=True)
+    state = models.CharField(max_length=50, null=True,
+                             blank=True, db_index=True)
+    when = models.DateTimeField(db_index=True)
+    microseconds = models.IntegerField(default=0)
+    publisher = models.CharField(max_length=50, null=True,
+                                 blank=True, db_index=True)
+    event = models.CharField(max_length=50, null=True,
+                                 blank=True, db_index=True)
+    service = models.CharField(max_length=50, null=True,
+                                 blank=True, db_index=True)
+    host = models.CharField(max_length=50, null=True,
+                                 blank=True, db_index=True)
+    instance = models.CharField(max_length=50, null=True,
+                                blank=True, db_index=True)
+
+
+class TenantForm(forms.ModelForm):
+    class Meta:
+        model = Tenant
+        fields = ('email', 'project_name')

stacktach/urls.py

@@ -0,0 +1,28 @@
+from django.conf.urls.defaults import patterns, include, url
+
+# Uncomment the next two lines to enable the admin:
+# from django.contrib import admin
+# admin.autodiscover()
+
+urlpatterns = patterns('',
+    url(r'^$', 'dss.stackmon.views.welcome', name='welcome'),
+    url(r'new_tenant', 'dss.stackmon.views.new_tenant', name='new_tenant'),
+    url(r'logout', 'dss.stackmon.views.logout', name='logout'),
+    url(r'^(?P<tenant_id>\d+)/$', 'dss.stackmon.views.home', name='home'),
+    url(r'^(?P<tenant_id>\d+)/data/$', 'dss.stackmon.views.data',
+        name='data'),
+    url(r'^(?P<tenant_id>\d+)/details/(?P<column>\w+)/(?P<row_id>\d+)/$',
+        'dss.stackmon.views.details', name='details'),
+    url(r'^(?P<tenant_id>\d+)/expand/(?P<row_id>\d+)/$',
+        'dss.stackmon.views.expand', name='expand'),
+    url(r'^(?P<tenant_id>\d+)/host_status/$', 
+        'dss.stackmon.views.host_status', name='host_status'),
+    url(r'^(?P<tenant_id>\d+)/instance_status/$',
+        'dss.stackmon.views.instance_status', name='instance_status'),
+
+    # Uncomment the admin/doc line below to enable admin documentation:
+    # url(r'^admin/doc/', include('django.contrib.admindocs.urls')),
+
+    # Uncomment the next line to enable the admin:
+    # url(r'^admin/', include(admin.site.urls)),
+)

stacktach/views.py

@@ -0,0 +1,249 @@
+# Copyright 2012 - Dark Secret Software Inc.
+
+from django.shortcuts import render_to_response
+from django import http
+from django import template
+from django.utils.functional import wraps
+
+from dss.stackmon import models
+
+import datetime
+import json
+import logging
+import pprint
+import random
+import sys
+
+logger = logging.getLogger(__name__)
+
+VERSION = 4
+
+
+class My401(BaseException):
+    pass
+
+
+class HttpResponseUnauthorized(http.HttpResponse):
+    status_code = 401
+
+
+def _monitor_message(routing_key, body):
+    event = body['event_type']
+    publisher = body['publisher_id']
+    parts = publisher.split('.')   
+    service = parts[0]
+    host = parts[1]
+    payload = body['payload']
+    request_spec = payload.get('request_spec', None)
+    instance = None
+    instance = payload.get('instance_id', instance)
+    nova_tenant = body.get('_context_project_id', None)
+    nova_tenant = payload.get('tenant_id', nova_tenant)
+    return dict(host=host, instance=instance, publisher=publisher,
+                service=service, event=event, nova_tenant=nova_tenant)
+                
+
+def _compute_update_message(routing_key, body):
+    publisher = None
+    instance = None
+    args = body['args']
+    host = args['host']
+    service = args['service_name']
+    event = body['method']
+    nova_tenant = args.get('_context_project_id', None)
+    return dict(host=host, instance=instance, publisher=publisher,
+                service=service, event=event, nova_tenant=nova_tenant)
+
+
+# routing_key : handler
+HANDLERS = {'monitor.info':_monitor_message,
+            'monitor.error':_monitor_message,
+            '':_compute_update_message}
+
+
+def _parse(tenant, args, json_args):
+    routing_key, body = args
+    handler = HANDLERS.get(routing_key, None)
+    if handler:
+        values = handler(routing_key, body)
+        if not values:
+            return {}
+
+        values['tenant'] = tenant
+        when = body['_context_timestamp']
+        when = datetime.datetime.strptime(when, "%Y-%m-%dT%H:%M:%S.%f")
+        values['when'] = when
+        values['microseconds'] = when.microsecond
+        values['routing_key'] = routing_key
+        values['json'] = json_args
+        record = models.RawData(**values)
+        record.save()
+        return values
+    return {}
+
+
+def _post_process_raw_data(rows, highlight=None):
+    for row in rows:
+        if "error" in row.routing_key:
+            row.is_error = True
+        if highlight and row.id == int(highlight):
+            row.highlight = True
+        row.when += datetime.timedelta(microseconds=row.microseconds)
+
+class State(object):
+    def __init__(self):
+        self.version = VERSION
+        self.tenant = None
+ 
+    def __str__(self):
+        tenant = "?"
+        if self.tenant:
+            tenant = "'%s' - %s (%d)" % (self.tenant.project_name,
+                                         self.tenant.email, self.tenant.id)
+        return "[Version %s, Tenant %s]" % (self.version, tenant)
+ 
+
+def _reset_state(request):
+    state = State()
+    request.session['state'] = state
+    return state
+
+   
+def _get_state(request, tenant_id=None):
+    tenant = None
+    if tenant_id:
+        try:
+            tenant = models.Tenant.objects.get(tenant_id=tenant_id)
+        except models.Tenant.DoesNotExist:
+            raise My401()
+
+    if 'state' in request.session:
+        state = request.session['state']
+    else:
+        state =_reset_state(request)
+
+    if hasattr(state, 'version') and state.version < VERSION:
+        state =_reset_state(request)
+        
+    state.tenant = tenant
+
+    return state
+
+
+def tenant_check(view):
+    @wraps(view)
+    def inner(*args, **kwargs):
+        try:
+            return view(*args, **kwargs)
+        # except HttpResponseUnauthorized, e:
+        except My401:
+            return HttpResponseUnauthorized()
+        
+    return inner
+
+
+def _default_context(state):
+    context = dict(utc=datetime.datetime.utcnow(), state=state)
+    return context
+
+    
+def welcome(request):
+    state = _reset_state(request, None)
+    return render_to_response('stackmon/welcome.html', _default_context(state))
+
+
+@tenant_check
+def home(request, tenant_id):
+    state = _get_state(request, tenant_id)
+    return render_to_response('stackmon/index.html', _default_context(state)) 
+
+
+def logout(request):
+    del request.session['state']
+    return render_to_response('stackmon/welcome.html', _default_context(None)) 
+
+
+@tenant_check
+def new_tenant(request):
+    state = _get_state(request)
+    context = _default_context(state)
+    if request.method == 'POST':
+        form = models.TenantForm(request.POST)
+        if form.is_valid():
+            rec = models.Tenant(**form.cleaned_data)
+            rec.save()
+            _reset_state(request, rec.tenant_id)
+            return http.HttpResponseRedirect('/stacktach/%d' % rec.tenant_id)
+    else:
+        form = models.TenantForm()
+        context['form'] = form
+    return render_to_response('stackmon/new_tenant.html', context)
+
+
+@tenant_check
+def data(request, tenant_id):
+    state = _get_state(request, tenant_id)
+    raw_args = request.POST.get('args', "{}")
+    args = json.loads(raw_args)
+    c = _default_context(state)
+    fields = _parse(state.tenant, args, raw_args)
+    c['cooked_args'] = fields
+    return render_to_response('stackmon/data.html', c)
+
+
+@tenant_check
+def details(request, tenant_id, column, row_id):
+    state = _get_state(request, tenant_id)
+    c = _default_context(state)
+    row = models.RawData.objects.get(pk=row_id)
+    value = getattr(row, column)
+    rows = models.RawData.objects.filter(tenant_id=tenant_id)
+    if column != 'when':
+        rows = rows.filter(**{column:value})
+    else:
+        from_time = value - datetime.timedelta(minutes=1)
+        to_time = value + datetime.timedelta(minutes=1)
+        rows = rows.filter(when__range=(from_time, to_time))
+                                  
+    rows = rows.order_by('-when', '-microseconds')[:200]
+    _post_process_raw_data(rows, highlight=row_id)
+    c['rows'] = rows
+    c['allow_expansion'] = True
+    c['show_absolute_time'] = True
+    return render_to_response('stackmon/rows.html', c)
+
+
+@tenant_check
+def expand(request, tenant_id, row_id):
+    state = _get_state(request, tenant_id)
+    c = _default_context(state)
+    row = models.RawData.objects.get(pk=row_id)
+    payload = json.loads(row.json)
+    pp = pprint.PrettyPrinter()
+    c['payload'] = pp.pformat(payload)
+    return render_to_response('stackmon/expand.html', c)
+
+
+@tenant_check
+def host_status(request, tenant_id):
+    state = _get_state(request, tenant_id)
+    c = _default_context(state)
+    hosts = models.RawData.objects.filter(tenant_id=tenant_id).\
+                                   filter(host__gt='').\
+                                   order_by('-when', '-microseconds')[:20]
+    _post_process_raw_data(hosts)
+    c['rows'] = hosts
+    return render_to_response('stackmon/host_status.html', c)
+
+
+@tenant_check
+def instance_status(request, tenant_id):
+    state = _get_state(request, tenant_id)
+    c = _default_context(state)
+    instances = models.RawData.objects.filter(tenant_id=tenant_id).\
+                                       exclude(instance='n/a').\
+                                       exclude(instance__isnull=True).\
+                                       order_by('-when', '-microseconds')[:20]
+    _post_process_raw_data(instances)
+    c['rows'] = instances
+    return render_to_response('stackmon/instance_status.html', c)

templates/base.html

@@ -0,0 +1,111 @@
+<!-- 
+Copyright 2012 - Dark Secret Software Inc.
+All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may
+not use this file except in compliance with the License. You may obtain
+a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+-->
+
+<html lang="en">
+    <head>                                                                  
+        <link href="http://ajax.googleapis.com/ajax/libs/jqueryui/1.8/themes/base/jquery-ui.css" rel="stylesheet" type="text/css"/>
+        <script src="http://ajax.googleapis.com/ajax/libs/jquery/1.5/jquery.min.js"></script>
+        <script src="http://ajax.googleapis.com/ajax/libs/jqueryui/1.8/jquery-ui.min.js"></script>
+
+        <script type="text/javascript" src="/static/js/jquery.timers.js"></script>
+        <link href='http://fonts.googleapis.com/css?family=Kaushan+Script' rel='stylesheet' type='text/css'>
+        <link href='http://fonts.googleapis.com/css?family=PT+Sans&subset=latin' rel='stylesheet' type='text/css'>
+
+        <style type="text/css">
+            .fancy {
+              font-family: 'Kaushan Script';
+              font-style: normal;
+              padding-bottom: 1em;
+              font-size: 3em;
+              color:#C63520;
+            }
+            h1, h2 {
+              font-family: 'PT Sans', serif;
+              font-style: normal;
+              letter-spacing: -0.076em;
+              line-height: 1em;
+            }
+            body {
+                font-size:75%;
+                color:#222;
+                background:#fff;
+                font-family:"Helvetica Neue", Arial, Helvetica, sans-serif;
+            }
+            a {
+                text-decoration:none;
+                color:#3f3f7f;
+            }
+            a:hover {
+                text-decoration:underline;
+            }
+            .cell-border {
+                border-left: 1px solid #bbbbcc;
+            }
+            .title {
+                font-weight: bold;
+
+            }
+            td {
+                padding-right: 1em;
+            }
+            .status {
+                border:1px #bbbbcc solid;
+                width:-1em;
+                margin-bottom:2em;
+            }
+            .status-title {
+                margin-left: 1em;
+                margin-right: 1em;
+                background-color: white;
+                font-family: 'PT Sans', serif;
+                color:#E5574D;
+                font-weight:bold;
+                font-size:1.5em;
+            }
+            .status-inner {
+                background-color:#fafaff;
+                padding-left:.5em;
+                margin-left:.5em;
+                margin-right:1em;
+                padding-bottom:1em;
+                margin-bottom:1em;
+            }
+            .std-height {
+                height:5em;
+                overflow-y:scroll;
+                overflow-x:hidden;
+            }
+
+        </style>
+
+        <script type="text/javascript">             
+            {% block extra_js %}
+            {% endblock %}
+            $(document).ready(function() {               
+            {% block extra_init_js %}
+            {% endblock %}
+            });
+       </script>                                                               
+    </head>                                                                 
+    <body>
+        <div class='fancy'>StackTach</div>
+
+{% block body %}
+{% endblock %}
+      
+    </body>                                                                 
+</html>

templates/data.html

@@ -0,0 +1 @@
+{{cooked_args|safe}}

templates/expand.html

@@ -0,0 +1,5 @@
+<div>
+<pre>
+{{payload|safe}}
+</pre>
+</div>

templates/host_status.html

@@ -0,0 +1,8 @@
+{% include "rows.html" %}
+
+<script type="text/javascript">
+  $(document).oneTime(2000, function() {
+    $('#host_activity').load('/{{state.tenant.tenant_id}}/host_status'); 
+  });
+</script>
+

templates/index.html

@@ -0,0 +1,50 @@
+{% extends "base.html" %}
+
+{% block extra_js %}
+            function details(tenant_id, column, row_id)
+            {
+                $("#detail").load('/' + tenant_id + '/details/' + column + '/' + row_id);
+            };
+
+            function expand(tenant_id, row_id)
+            {
+                $("#row_expansion_" + row_id).load('/' + tenant_id + '/expand/' + row_id);
+            };
+{% endblock %}
+
+{% block extra_init_js %}            
+                $('#host-box').resizable();
+                $('#instance-box').resizable();
+{% endblock %}                
+
+{% block body %}
+        <div style='float:right;'>{{state.tenant.email}} (TID:{{state.tenant.tenant_id}}) - {{state.tenant.project_name}} <a href='/logout'>logout</a></div>
+        <div class='status-title'>Recent Host Activity</div>
+        <div id='host-box' class='status std-height'>
+            <div id='host_activity' class='status-inner'>
+                {% include "host_status.html" %}
+            </div>
+        </div>
+
+        <div class='status-title'>Recent Instance Activity</div>
+        <div id='instance-box' class='status std-height'>
+            <div id='instance_activity' class='status-inner'>
+                {% include "instance_status.html" %}
+           </div>
+        </div>
+        <!--
+        <div class='status-title'>Commands</div>
+        <div class='status'>
+            <div class='status-inner'>
+                <input type=text style='width:100%; background-color:black;color:white;'
+                        value='<custom query here>'/>
+            </div>
+        </div>
+        -->
+        <div class='status-title'>Details</div>
+        <div class='status'>
+            <div id='detail' class='status-inner'>
+                <div>click on an item above to see more of the same type.</div>
+            </div>
+        </div>
+{% endblock %}        

templates/instance_status.html

@@ -0,0 +1,8 @@
+{% include "rows.html" %}
+
+<script type="text/javascript">
+  $(document).oneTime(2000, function() {
+    $('#instance_activity').load('/{{state.tenant.tenant_id}}/instance_status'); 
+  });
+</script>
+

templates/new_tenant.html

@@ -0,0 +1,13 @@
+{% extends "base.html" %}
+
+{% block body %}
+    <div class='status-title'>New Tenant</div>
+    <div id='host-box' class='status'>
+        <div id='host_activity' class='status-inner'>
+        <form action='/new_tenant/' method='post'>{% csrf_token %}
+        {{ form.as_p }}
+        <input type="submit" value="Submit" />
+        </form>
+        </div>
+    </div>
+{% endblock %}       

templates/rows.html

@@ -0,0 +1,50 @@
+<table style='font-size:1em;'>
+<th>
+    <td class='title'></td>
+    <td class='title'>source</td>
+    <td class='title'>tenant</td>
+    <td class='title'>service</td>
+    <td class='title'>host</td>
+    <td class='title'>event</td>
+    <td class='title'>instance</td>
+    <td class='title'>when</td>
+</th>
+{% for row in rows %}
+<tr {% if row.highlight %}style='background-color:#FFD88F;'{% endif %} >
+    <td>
+    {% if allow_expansion %}
+    <a href='#' onclick='expand({{state.tenant.tenant_id}}, {{row.id}});'>[+]</a>
+    {% endif %}
+    </td>
+
+    <td>
+    <td><span style='{% if row.is_error %}background-color:#ffaaaa;{% endif %}'>
+    <a href='#' onclick='details({{state.tenant.tenant_id}}, "routing_key", {{row.id}});'>{{row.routing_key}}</a>
+    </span>
+    </td>
+    <td class='cell-border'>
+        <a href='#' onclick='details({{state.tenant.tenant_id}}, "nova_tenant", {{row.id}});'>
+        {% if row.nova_tenant %}{{row.nova_tenant}}{% endif %}
+        </a>
+    </td>
+    <td class='cell-border'><a href='#' onclick='details({{state.tenant.tenant_id}}, "service", {{row.id}});'>{{row.service}}</a></td>
+    <td class='cell-border'><a href='#' onclick='details({{state.tenant.tenant_id}}, "host", {{row.id}});'>{{row.host}}</a></td>
+    <td class='cell-border'><b><a href='#' onclick='details({{state.tenant.tenant_id}}, "event", {{row.id}});'>{{row.event}}</a></b></td>
+    <td class='cell-border'>
+        <a href='#' onclick='details({{state.tenant.tenant_id}}, "instance", {{row.id}});'>
+        {% if row.instance %}
+            {{row.instance}}
+        {% endif %}
+        </a>
+    </td>
+    <td class='cell-border'><a href='#' onclick='details({{state.tenant.tenant_id}}, "when", {{row.id}});'>{% if show_absolute_time %}{{row.when}}{%else%}{{row.when|timesince:utc}} ago{%endif%}</a></td>
+</tr>
+{% if allow_expansion %}
+<tr>
+<td colspan=8>
+<div id='row_expansion_{{row.id}}' style='font-size:1.2em'></div>
+</td>
+</tr>
+{% endif %}
+{% endfor %}
+</table>

templates/welcome.html

@@ -0,0 +1,24 @@
+{% extends "base.html" %}
+
+{% block body %}
+    <div class='status-title'>About</div>
+    <div id='host-box' class='status'>
+        <div id='host_activity' class='status-inner'>
+        StackTach is a hosted debug/monitoring tool for OpenStack Nova
+        deployments.
+        </div>
+    </div>
+
+    <div class='status-title'>Connecting StackTach to OpenStack</div>
+    <div id='instance-box' class='status'>
+        <div id='instance_activity' class='status-inner'>
+            <ul>
+            <li>Get a <a href='/new_tenant'>StackTach Tenant ID</a>
+            <li>Add <pre>--notification_driver=nova.notifier.rabbit_notifier</pre> and 
+            <li><pre>--notification_topics=monitor</pre> to your nova.conf file.
+            <li>Configure and run the <a target='_blank' href='https://github.com/SandyWalsh/StackTach'>StackTach Worker</a> somewhere in your Nova development environment. 
+            <li>Restart Nova and visit http://darksecretsoftware.com/stacktach/[your_tenant_id]/ to see your Nova installation in action!
+            </ul>
+        </div>
+    </div>
+{% endblock %}       

urls.py

@@ -0,0 +1,15 @@
+from django.conf.urls.defaults import patterns, include, url
+
+# Uncomment the next two lines to enable the admin:
+# from django.contrib import admin
+# admin.autodiscover()
+
+urlpatterns = patterns('',
+    url(r'^', include('stacktach.urls')),
+
+    # Uncomment the admin/doc line below to enable admin documentation:
+    # url(r'^admin/doc/', include('django.contrib.admindocs.urls')),
+
+    # Uncomment the next line to enable the admin:
+    # url(r'^admin/', include(admin.site.urls)),
+)

worker.py

@@ -1,4 +1,20 @@
 # Copyright 2012 - Dark Secret Software Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This is the worker you run in your OpenStack environment. You need
+# to set TENANT_ID and URL to point to your StackTach web server.
 
 import json
 import kombu
@@ -9,7 +25,9 @@ import threading
 import urllib
 import urllib2
 
-url = 'http://darksecretsoftware.com/stacktach/data/'
+# CHANGE THESE FOR YOUR INSTALLATION ...
+TENANT_ID = 1
+URL = 'http://darksecretsoftware.com/stacktach/%d/data/' % TENANT_ID
 
 # For now we'll just grab all the fanout messages from compute to scheduler ...
 scheduler_exchange = kombu.entity.Exchange("scheduler_fanout", type="fanout",
@@ -57,18 +75,21 @@ class SchedulerFanoutConsumer(kombu.mixins.ConsumerMixin):
         try:
             raw_data = dict(args=jvalues)
             cooked_data = urllib.urlencode(raw_data)
-            req = urllib2.Request(url, cooked_data)
+            req = urllib2.Request(URL, cooked_data)
             response = urllib2.urlopen(req)
             page = response.read()
             print page
         except urllib2.HTTPError, e:
+            if e.code == 401:
+                print "Unauthorized. Correct tenant id of %d?" % TENANT_ID
             print e
             page = e.read()
             print page
             raise
 
     def on_scheduler(self, body, message):
-        self._process(body, message)
+        # Uncomment if you want periodic compute node status updates.
+        # self._process(body, message)
         message.ack()
 
     def on_nova(self, body, message):