Aggregated stats for roles

Making aggregated stats for roles. Extracting more code to
metering utils, to avoid duplication.

Depends on Ceilometer patch:
https://review.openstack.org/#/c/119803/

Change-Id: I6a9e16e8305ef2d6f14912081059ec93ea0606f1

tuskar_ui/infrastructure/nodes/views.py

@@ -100,103 +100,26 @@ class DetailView(horizon_tabs.TabView):
 
 
 class PerformanceView(base.TemplateView):
-    LABELS = {
-        'hardware.cpu.load.1min': _("CPU load 1 min average"),
-        'hardware.system_stats.cpu.util': _("CPU utilization"),
-        'hardware.system_stats.io.outgoing.blocks': _("IO raw sent"),
-        'hardware.system_stats.io.incoming.blocks': _("IO raw received"),
-        'hardware.network.ip.outgoing.datagrams': _("IP out requests"),
-        'hardware.network.ip.incoming.datagrams': _("IP in requests"),
-        'hardware.memory.swap.util': _("Swap utilization"),
-    }
-
-    @staticmethod
-    def _series_for_meter(aggregates,
-                          meter_id,
-                          meter_name,
-                          stats_name,
-                          unit):
-        """Construct datapoint series for a meter from resource aggregates."""
-        series = []
-        for resource in aggregates:
-            if resource.get_meter(meter_name):
-                name = PerformanceView.LABELS.get(meter_id, meter_name)
-                point = {'unit': unit,
-                         'name': unicode(name),
-                         'meter': meter_id,
-                         'data': []}
-                for statistic in resource.get_meter(meter_name):
-                    date = statistic.duration_end[:19]
-                    value = float(getattr(statistic, stats_name))
-                    point['data'].append({'x': date, 'y': value})
-                series.append(point)
-        return series
-
     def get(self, request, *args, **kwargs):
         meter = request.GET.get('meter')
         date_options = request.GET.get('date_options')
         date_from = request.GET.get('date_from')
         date_to = request.GET.get('date_to')
         stats_attr = request.GET.get('stats_attr', 'avg')
-        group_by = request.GET.get('group_by')
         barchart = bool(request.GET.get('barchart'))
 
         node_uuid = kwargs.get('node_uuid')
         node = api.node.Node.get(request, node_uuid)
 
-        unit = ''
-        series = []
-
         try:
             instance_uuid = node.instance_uuid
         except AttributeError:
-            pass
+            json_output = None
         else:
-            query = [{'field': 'resource_id',
+            json_output = metering_utils.get_nodes_stats(
-                      'op': 'eq',
+                request, instance_uuid, meter, date_options=date_options,
-                      'value': instance_uuid}]
+                date_from=date_from, date_to=date_to, stats_attr=stats_attr,
-
+                barchart=barchart)
-            # Disk and Network I/O: data from 2 meters in one chart
-            if meter == 'disk-io':
-                meters = metering_utils.get_meters([
-                    'hardware.system_stats.io.outgoing.blocks',
-                    'hardware.system_stats.io.incoming.blocks'
-                ])
-            elif meter == 'network-io':
-                meters = metering_utils.get_meters([
-                    'hardware.network.ip.outgoing.datagrams',
-                    'hardware.network.ip.incoming.datagrams'
-                ])
-            else:
-                meters = metering_utils.get_meters([meter])
-
-            date_from, date_to = metering_utils._calc_date_args(
-                date_from,
-                date_to,
-                date_options)
-
-            for meter_id, meter_name in meters:
-                resources, unit = metering_utils.query_data(
-                    request=request,
-                    date_from=date_from,
-                    date_to=date_to,
-                    group_by=group_by,
-                    meter=meter_id,
-                    query=query)
-                serie = self._series_for_meter(
-                    resources,
-                    meter_id,
-                    meter_name,
-                    stats_attr,
-                    unit)
-                series += serie
-
-        json_output = metering_utils.create_json_output(
-            series,
-            barchart,
-            unit,
-            date_from,
-            date_to)
 
         return http.HttpResponse(json.dumps(json_output),
                                  content_type='application/json')

tuskar_ui/infrastructure/roles/templates/roles/detail.html

@@ -8,24 +8,114 @@
 {% endblock page_header %}
 
 {% block main %}
+
 <div class="row">
-  <div class="col-xs-12">
+  <div class="col-xs-3">
     <p><strong>{% blocktrans count counter=nodes|length %}{{ counter }} instance{% plural %}{{ counter }} instances{% endblocktrans %}</strong></p>
     <dl>
-    <dt>{% trans 'Flavor' %}</dt>
+      <dt>{% trans 'Flavor' %}</dt>
-    {% if flavor %}
+      {% if flavor %}
-      <dd><em>{{ flavor.name }}</em> {{ flavor.get_keys.cpu_arch }} | {{ flavor.vcpus }} {% trans "CPU" %} | {{ flavor.ram }} {% trans "MB RAM" %} | {{ flavor.disk }} {% trans "GB HDD" %}</dd>
+        <dd><em>{{ flavor.name }}</em> {{ flavor.get_keys.cpu_arch }} | {{ flavor.vcpus }} {% trans "CPU" %} | {{ flavor.ram }} {% trans "MB RAM" %} | {{ flavor.disk }} {% trans "GB HDD" %}</dd>
-    {% else %}
+      {% else %}
-      <dd>{% trans 'No flavor associated' %}</dd>
+        <dd>{% trans 'No flavor associated' %}</dd>
-    {% endif %}
+      {% endif %}
-    <dt>{% trans 'Image' %}</dt>
+      <dt>{% trans 'Image' %}</dt>
-    {% if image %}
+      {% if image %}
-      <dd>{{ image.name }}</dd>
+        <dd>{{ image.name }}</dd>
-    {% else %}
+      {% else %}
-      <dd>{% trans 'No image associated' %}</dd>
+        <dd>{% trans 'No image associated' %}</dd>
-    {% endif %}
+      {% endif %}
     </dl>
+  </div>
+  <div class="col-xs-9">
+    {% if meter_conf %}
+      {% url 'horizon:infrastructure:roles:performance' role.uuid as role_perf_url %}
+      <div id="ceilometer-stats" class="row">
+        <form class="form-horizontal" id="linechart_general_form">
+          <div class="form-group">
+            <label for="date_options" class="control-label col-sm-3">{% trans "Period" %}:&nbsp;</label>
+            <div class="col-sm-4">
+              <select data-line-chart-command="select_box_change"
+                      id="date_options"
+                      name="date_options"
+                      class="form-control">
+                <option value="0.041666">{% trans "Last hour" %}</option>
+                <option value="0.25">{% trans "Last 6 hours" %}</option>
+                <option value="0.5">{% trans "Last 12 hours" %}</option>
+                <option value="1" selected="selected">{% trans "Last day" %}</option>
+                <option value="7">{% trans "Last week" %}</option>
+                <option value="{% now 'j' %}">{% trans "Month to date" %}</option>
+                <option value="15">{% trans "Last 15 days" %}</option>
+                <option value="30">{% trans "Last 30 days" %}</option>
+                <option value="365">{% trans "Last year" %}</option>
+                <option value="other">{% trans "Other" %}</option>
+              </select>
+            </div>
+          </div>
+          <div class="form-group" id="date_from_group">
+            <label for="date_from" class="control-label col-sm-3">{% trans "From" %}:&nbsp;</label>
+            <div class="col-sm-4">
+              <input data-line-chart-command="date_picker_change"
+                     type="text"
+                     id="date_from"
+                     name="date_from"
+                     class="form-control"/>
+            </div>
+          </div>
+          <div class="form-group" id="date_to_group">
+            <label for="date_to" class="control-label col-sm-3">{% trans "To" %}:&nbsp;</label>
+            <div class="col-sm-4">
+              <input data-line-chart-command="date_picker_change"
+                     type="text"
+                     id="date_to"
+                     name="date_to"
+                     class="form-control"/>
+            </div>
+          </div>
+        </form>
+      </div>
 
+      <script type="text/javascript">
+        if (typeof $ !== 'undefined') {
+          show_hide_datepickers();
+        } else {
+          addHorizonLoadEvent(function() {
+            show_hide_datepickers();
+          });
+        }
+
+        function show_hide_datepickers() {
+          var date_options = $("#date_options");
+          date_options.change(function(evt) {
+            if ($(this).find("option:selected").val() === "other") {
+              evt.stopPropagation();
+              $("#date_from, #date_to").val('');
+              $("#date_from_group, #date_to_group").show();
+            } else {
+              $("#date_from_group, #date_to_group").hide();
+            }
+          });
+          if (date_options.find("option:selected").val() === "other") {
+            $("#date_from_group, #date_to_group").show();
+          } else {
+            $("#date_from_group, #date_to_group").hide();
+          }
+        }
+      </script>
+      <div id="node-charts" class="row">
+        {% for meter_label, url_part, y_max in meter_conf %}
+        <div class="col-lg-4">
+          {% include "infrastructure/_performance_chart.html" with label=meter_label y_max=y_max url=role_perf_url|add:"?"|add:url_part only %}
+        </div>
+        {% endfor %}
+      </div>
+    {% else %}
+      <p>{% trans 'Metering service is not enabled.' %}</p>
+    {% endif %}
+  </div>
+</div>
+<div class="row">
+  <div class="col-xs-12">
     {{ table.render }}
   </div>
 </div>

tuskar_ui/infrastructure/roles/urls.py

@@ -24,5 +24,6 @@ urlpatterns = urls.patterns(
              name='detail'),
     urls.url(r'^(?P<role_id>[^/]+)/edit$', views.UpdateView.as_view(),
              name='update'),
-
+    urls.url(r'^(?P<role_id>[^/]+)/performance/$',
+             views.PerformanceView.as_view(), name='performance'),
 )

tuskar_ui/infrastructure/roles/views.py

@@ -11,10 +11,13 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
+import json
 
 from django.core.urlresolvers import reverse
 from django.core.urlresolvers import reverse_lazy
+from django import http
 from django.utils.translation import ugettext_lazy as _
+from django.views.generic import base
 
 from glanceclient import exc as glance_exc
 from horizon import exceptions as horizon_exceptions
@@ -22,9 +25,12 @@ from horizon import tables as horizon_tables
 from horizon import utils
 from horizon import workflows
 
+from openstack_dashboard.api import base as api_base
+
 from tuskar_ui import api
 from tuskar_ui.infrastructure.roles import tables
 from tuskar_ui.infrastructure.roles import workflows as role_workflows
+from tuskar_ui.utils import metering as metering_utils
 
 
 INDEX_URL = 'horizon:infrastructure:roles:index'
@@ -123,6 +129,23 @@ class DetailView(horizon_tables.DataTableView, RoleMixin, StackMixin):
         # won't work right now
         context['image'] = role.image(plan)
 
+        if stack:
+            if api_base.is_service_enabled(self.request, 'metering'):
+                # Meter configuration in the following format:
+                # (meter label, url part, barchart (True/False))
+                context['meter_conf'] = (
+                    (_('System Load'),
+                     metering_utils.url_part('hardware.cpu.load.1min', False),
+                     None),
+                    (_('CPU Utilization'),
+                     metering_utils.url_part('hardware.system_stats.cpu.util',
+                                             True),
+                     '100'),
+                    (_('Swap Utilization'),
+                     metering_utils.url_part('hardware.memory.swap.util',
+                                             True),
+                     '100'),
+                )
         return context
 
 
@@ -157,3 +180,31 @@ class UpdateView(workflows.WorkflowView):
                 'flavor': role_flavor,
                 'image': role_image,
                 }
+
+
+class PerformanceView(base.TemplateView, RoleMixin, StackMixin):
+    def get(self, request, *args, **kwargs):
+        meter = request.GET.get('meter')
+        date_options = request.GET.get('date_options')
+        date_from = request.GET.get('date_from')
+        date_to = request.GET.get('date_to')
+        stats_attr = request.GET.get('stats_attr', 'avg')
+        barchart = bool(request.GET.get('barchart'))
+
+        plan = api.tuskar.Plan.get_the_plan(self.request)
+        role = self.get_role(None)
+        role.image(plan)
+
+        try:
+            image = role.image(plan)
+            image_uuid = image.id
+        except AttributeError:
+            json_output = None
+        else:
+            json_output = metering_utils.get_nodes_stats(
+                request, image_uuid, meter, date_options=date_options,
+                date_from=date_from, date_to=date_to, stats_attr=stats_attr,
+                barchart=barchart, group_by='image_id')
+
+        return http.HttpResponse(json.dumps(json_output),
+                                 mimetype='application/json')

tuskar_ui/utils/metering.py

@@ -57,6 +57,16 @@ SETTINGS = {
     }
 }
 
+LABELS = {
+    'hardware.cpu.load.1min': _("CPU load 1 min average"),
+    'hardware.system_stats.cpu.util': _("CPU utilization"),
+    'hardware.system_stats.io.outgoing.blocks': _("IO raw sent"),
+    'hardware.system_stats.io.incoming.blocks': _("IO raw received"),
+    'hardware.network.ip.outgoing.datagrams': _("IP out requests"),
+    'hardware.network.ip.incoming.datagrams': _("IP in requests"),
+    'hardware.memory.swap.util': _("Swap utilization"),
+}
+
 
 #TODO(lsmola) this should probably live in Horizon common
 def query_data(request,
@@ -89,20 +99,21 @@ def query_data(request,
         unit = meter_list[0].unit
     except Exception:
         unit = ""
-    if group_by == "resources":
+
-        # TODO(lsmola) need to implement group_by groups of resources
+    ceilometer_usage = ceilometer.CeilometerUsage(request)
-        resources = []
+    try:
-        unit = ""
+        if group_by:
-    else:
+            resources = ceilometer_usage.resource_aggregates_with_statistics(
-        ceilometer_usage = ceilometer.CeilometerUsage(request)
+                query, [meter], period=period, stats_attr=None,
-        try:
+                additional_query=additional_query)
+        else:
             resources = ceilometer_usage.resources_with_statistics(
                 query, [meter], period=period, stats_attr=None,
                 additional_query=additional_query)
-        except Exception:
+    except Exception:
-            resources = []
+        resources = []
-            exceptions.handle(request,
+        exceptions.handle(request,
-                              _('Unable to retrieve statistics.'))
+                          _('Unable to retrieve statistics.'))
     return resources, unit
 
 
@@ -217,3 +228,89 @@ def create_json_output(series, barchart, unit, date_from, date_to):
     json_output = {'series': series}
     json_output = dict(json_output.items() + settings.items())
     return json_output
+
+
+def _series_for_meter(aggregates,
+                      meter_id,
+                      meter_name,
+                      stats_name,
+                      unit):
+    """Construct datapoint series for a meter from resource aggregates."""
+    series = []
+    for resource in aggregates:
+        if resource.get_meter(meter_name):
+            name = LABELS.get(meter_id, meter_name)
+            point = {'unit': unit,
+                     'name': unicode(name),
+                     'meter': meter_id,
+                     'data': []}
+            for statistic in resource.get_meter(meter_name):
+                date = statistic.duration_end[:19]
+                value = float(getattr(statistic, stats_name))
+                point['data'].append({'x': date, 'y': value})
+            series.append(point)
+    return series
+
+
+def get_nodes_stats(request, uuid, meter, date_options=None, date_from=None,
+                    date_to=None, stats_attr=None, barchart=None,
+                    group_by=None):
+
+    unit = ''
+    series = []
+
+    if uuid:
+        if group_by == "image_id":
+            query = {}
+            image_query = [{"field": "metadata.%s" % group_by,
+                            "op": "eq",
+                            "value": uuid}]
+            query[uuid] = image_query
+        else:
+            query = [{'field': 'resource_id',
+                      'op': 'eq',
+                      'value': uuid}]
+
+    # Disk and Network I/O: data from 2 meters in one chart
+    if meter == 'disk-io':
+        meters = get_meters([
+            'hardware.system_stats.io.outgoing.blocks',
+            'hardware.system_stats.io.incoming.blocks'
+        ])
+    elif meter == 'network-io':
+        meters = get_meters([
+            'hardware.network.ip.outgoing.datagrams',
+            'hardware.network.ip.incoming.datagrams'
+        ])
+    else:
+        meters = get_meters([meter])
+
+    date_from, date_to = _calc_date_args(
+        date_from,
+        date_to,
+        date_options)
+
+    for meter_id, meter_name in meters:
+        resources, unit = query_data(
+            request=request,
+            date_from=date_from,
+            date_to=date_to,
+            group_by=group_by,
+            meter=meter_id,
+            query=query)
+        serie = _series_for_meter(
+            resources,
+            meter_id,
+            meter_name,
+            stats_attr,
+            unit)
+        series += serie
+
+    json_output = create_json_output(
+        series,
+        barchart,
+        unit,
+        date_from,
+        date_to)
+
+    return json_output