Implements stats on blueprint mentions

Change-Id: I531aa5b723f18f5b65587dd04bd29212100cffaf

dashboard/static/css/style.css

@@ -307,3 +307,7 @@ a[href^="https://launchpad"]:after {
 .importanceUndecided, .importanceUndecided a {
     color: #999999;
 }
+
+.select2-results {
+    max-height: 300px;
+}

dashboard/static/js/stackalytics-ui.js

@@ -83,12 +83,12 @@ function renderTimeline(options) {
     });
 }
 
-function renderTableAndChart(url, container_id, table_id, chart_id, link_param, options) {
+function renderTableAndChart(url, container_id, table_id, chart_id, link_param, table_column_names) {
 
     $(document).ready(function () {
 
         $.ajax({
-            url: make_uri(url, options),
+            url: make_uri(url),
             dataType: "json",
             success: function (data) {
 
@@ -99,7 +99,6 @@ function renderTableAndChart(url, container_id, table_id, chart_id, link_param,
                 var aggregate = 0;
                 var index = 1;
                 var i;
-                var hasComment = false;
 
                 data = data["stats"];
 
@@ -121,18 +120,15 @@ function renderTableAndChart(url, container_id, table_id, chart_id, link_param,
                     } else {
                         index++;
                     }
-                    var link;
+                    if (!data[i].link) {
                         if (data[i].id) {
-                        link = make_link(data[i].id, data[i].name, link_param);
+                            data[i]["link"] = make_link(data[i].id, data[i].name, link_param);
                         } else {
-                        link = data[i].name
+                            data[i]["link"] = data[i].name
                         }
-                    var rec = {"index": index_label, "link": link, "metric": data[i].metric};
-                    if (data[i].comment) {
-                        rec["comment"] = data[i].comment;
-                        hasComment = true;
                     }
-                    tableData.push(rec);
+                    data[i]["index"] = index_label;
+                    tableData.push(data[i]);
                 }
 
                 if (i == limit) {
@@ -141,26 +137,29 @@ function renderTableAndChart(url, container_id, table_id, chart_id, link_param,
                     chartData.push(["others", aggregate]);
                 }
 
-                var tableColumns = [
+                if (!table_column_names) {
-                    { "mData": "index" },
+                    table_column_names = ["index", "link", "metric"];
-                    { "mData": "link" },
+                }
-                    { "mData": "metric" }
+                var tableColumns = [];
-                ];
+                var sort_by_column = 0;
-                if (hasComment) {
+                for (i = 0; i < table_column_names.length; i++) {
-                    tableColumns.push({ "mData": "comment"})
+                    tableColumns.push({"mData": table_column_names[i]})
+                    if (table_column_names[i] == "metric") {
+                        sort_by_column = i;
+                    }
                 }
 
                 if (table_id) {
                     $("#" + table_id).dataTable({
                         "aLengthMenu": [
-                            [25, 50, -1],
+                            [10, 25, 50, -1],
-                            [25, 50, "All"]
+                            [10, 25, 50, "All"]
                         ],
                         "aaSorting": [
-                            [ 2, "desc" ]
+                            [ sort_by_column, "desc" ]
                         ],
                         "sPaginationType": "full_numbers",
-                        "iDisplayLength": 25,
+                        "iDisplayLength": 10,
                         "aaData": tableData,
                         "aoColumns": tableColumns
                     });

dashboard/templates/layout.html

@@ -19,10 +19,10 @@
 
 <link rel="icon" href="{{ url_for('static', filename='images/favicon.png') }}" type="image/png"/>
 
-<link rel=stylesheet type=text/css href="{{ url_for('static', filename='css/style.css') }}">
 <link rel=stylesheet type=text/css href="{{ url_for('static', filename='css/jquery.jqplot.min.css') }}">
 <link rel=stylesheet type=text/css href="{{ url_for('static', filename='css/jquery.dataTables.css') }}">
 <link rel=stylesheet type=text/css href="{{ url_for('static', filename='css/select2.css') }}">
+<link rel=stylesheet type=text/css href="{{ url_for('static', filename='css/style.css') }}">
 
 <script type="text/javascript" src="{{ url_for('static', filename='js/jquery-1.9.1.min.js') }}"></script>
 <script type="text/javascript" src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>

dashboard/templates/overview.html

@@ -2,7 +2,7 @@
 
 {% set show_company_breakdown = (not company) and (not user_id) %}
 {% set show_engineer_breakdown = (not user_id) %}
-{% set show_bp_breakdown = (module) and (not user_id) and (metric == 'bpd!') %}
+{% set show_bp_breakdown = (metric in ['bpd', 'bpc']) %}
 {% set show_module_breakdown = (not module) %}
 {% set show_user_activity = (user_id) %}
 {% set show_module_activity = (module) and (not user_id) %}
@@ -12,6 +12,7 @@
 {% set show_contribution = (show_user_contribution) or (show_module_contribution) %}
 {% set show_user_profile = (user_id) %}
 {% set show_top_mentors_options = (metric == 'tm_marks') %}
+{% set show_review_ratio = (metric in ['marks', 'tm_marks']) %}
 
 {% block scripts %}
 <script type="text/javascript">
@@ -22,10 +23,16 @@
     renderTableAndChart("/api/1.0/stats/companies", "company_container", "company_table", "company_chart", "company");
     {% endif %}
     {% if show_engineer_breakdown %}
-    renderTableAndChart("/api/1.0/stats/engineers", "engineer_container", "engineer_table", "engineer_chart", "user_id", {company: "{{ company|encode }}" });
+        {% if show_review_ratio %}
+        renderTableAndChart("/api/1.0/stats/engineers", "engineer_container", "engineer_table", "engineer_chart", "user_id",
+                ["index", "link", "metric", "mark_ratio"]);
+        {% else %}
+        renderTableAndChart("/api/1.0/stats/engineers", "engineer_container", "engineer_table", "engineer_chart", "user_id");
+        {% endif %}
     {% endif %}
     {% if show_bp_breakdown %}
-    renderTableAndChart("/api/1.0/stats/bpd", "bp_container", "bp_table", "bp_chart", "bpd", {module: "{{ module|encode }}" });
+    renderTableAndChart("/api/1.0/stats/bp", "bp_container", "bp_table", "bp_chart", "name",
+            ["index", "link", "status", "date", "metric"]);
     {% endif %}
     {% if show_module_breakdown %}
     renderTableAndChart("/api/1.0/stats/modules", "module_container", "module_table", "module_chart", "module");
@@ -135,7 +142,7 @@
             </div>
             {%/if%}
         {%elif ((record_type == "bpd") || (record_type == "bpc")) %}
-            <div style='font-weight: bold;'>${title} (<a href="/report/blueprint/${module}/${name}" class="ext_link">${name}</a>)</div>
+            <div style='font-weight: bold;'>${title} ({%html blueprint_link %})</div>
             <div style='white-space: pre-wrap;'>${summary}</div>
 
             <div>Priority: <span class="specpriority${priority}">${priority}</span></div>
@@ -205,7 +212,7 @@
             });
         </script>
         <div class="drop" style="margin-top: 1em;">
-            <label for="review_nth">Review #</label>
+            <label for="review_nth">Aggregate stats for first # reviews</label>
             <select id="review_nth" name="review_nth"
                     style="min-width: 140px;"
                     data-placeholder="Select review #">
@@ -266,27 +273,6 @@
     </div>
     {% endif %}
 
-    {% if show_bp_breakdown %}
-    <div id="bp_container">
-    <h2>Mention # by blueprint</h2>
-
-    <div id="bp_chart" style="width: 100%; height: 350px;"></div>
-
-    <table id="bp_table" class="display">
-        <thead>
-            <tr>
-                <th>#</th>
-                <th>Blueprint</th>
-                <th>Mention #</th>
-            </tr>
-        </thead>
-        <tbody>
-        </tbody>
-    </table>
-    <div class="spacer"></div>
-    </div>
-    {% endif %}
-
     {% if show_user_profile %}
         <div id="user_profile_container" style="margin-bottom: 2em;"></div>
     {% endif %}
@@ -330,6 +316,29 @@
     </div>
     {% endif %}
 
+    {% if show_bp_breakdown %}
+    <div id="bp_container">
+    <h2>Mention # by blueprint</h2>
+
+    <div id="bp_chart" style="width: 100%; height: 350px;"></div>
+
+    <table id="bp_table" class="display">
+        <thead>
+            <tr>
+                <th>#</th>
+                <th>Blueprint</th>
+                <th>Status</th>
+                <th>Date</th>
+                <th>Mentions</th>
+            </tr>
+        </thead>
+        <tbody>
+        </tbody>
+    </table>
+    <div class="spacer"></div>
+    </div>
+    {% endif %}
+
     {% if show_user_contribution %}
         <div id="contribution_container"></div>
     {% endif %}

dashboard/web.py

@@ -405,7 +405,7 @@ def aggregate_filter():
                     else:
                         mark_distribution.append('0')
 
-                new_record['comment'] = (
+                new_record['mark_ratio'] = (
                     '|'.join(mark_distribution) +
                     ' (%.1f%%)' % ((positive * 100.0) / record['metric']))
                 return new_record
@@ -685,6 +685,8 @@ def get_activity_json(records):
             if 'mention_date' in record:
                 blueprint['mention_date_str'] = format_datetime(
                     record['mention_date'])
+            blueprint['blueprint_link'] = make_blueprint_link(
+                blueprint['name'], blueprint['module'])
             result.append(blueprint)
 
     result.sort(key=lambda x: x['date'], reverse=True)
@@ -800,35 +802,24 @@ def get_module(module):
     flask.abort(404)
 
 
-@app.route('/api/1.0/stats/bpd/<module>')
+@app.route('/api/1.0/stats/bp')
 @jsonify('stats')
 @exception_handler()
-def get_bpd(module):
+@record_filter()
-    memory_storage_inst = get_vault()['memory_storage']
+def get_bpd(records):
-
-    module = module.lower()
-    record_ids = (
-        set(memory_storage_inst.get_record_ids_by_type('bpd')) &
-        set(memory_storage_inst.get_record_ids_by_modules([module])))
-
-    # param = get_parameter(kwargs, 'release', 'releases', use_default)
-    # if param:
-    #     if 'all' not in param:
-    #         record_ids &= (
-    #             memory_storage.get_record_ids_by_releases(
-    #                 c.lower() for c in param))
-
     result = []
-    for record in memory_storage_inst.get_records(record_ids):
+    for record in records:
+        if record['record_type'] in ['bpd', 'bpc']:
             result.append({
-            'date': record['date'],
+                'date': format_date(record['date']),
-            'lifecycle_status': record['lifecycle_status'],
+                'status': record['lifecycle_status'],
                 'metric': record['mention_count'],
                 'id': record['name'],
                 'name': record['name'],
+                'link': make_blueprint_link(record['name'], record['module'])
             })
 
-    result.sort(key=lambda x: x['metric'])
+    result.sort(key=lambda x: x['metric'], reverse=True)
 
     return result
 
@@ -996,6 +987,10 @@ def format_datetime(timestamp):
         timestamp).strftime('%d %b %Y %H:%M:%S')
 
 
+def format_date(timestamp):
+    return datetime.datetime.utcfromtimestamp(timestamp).strftime('%d-%b-%y')
+
+
 @app.template_filter('launchpadmodule')
 def format_launchpad_module_link(module):
     return '<a href="https://launchpad.net/%s">%s</a>' % (module, module)
@@ -1023,6 +1018,11 @@ def make_link(title, uri=None, options=None):
     return '<a href="%(uri)s">%(title)s</a>' % {'uri': uri, 'title': title}
 
 
+def make_blueprint_link(name, module):
+    uri = '/report/blueprint/' + module + '/' + name
+    return '<a href="%(uri)s">%(title)s</a>' % {'uri': uri, 'title': name}
+
+
 def make_commit_message(record):
     s = record['message']
     module = record['module']

stackalytics/processor/record_processor.py

@@ -386,7 +386,7 @@ class RecordProcessor(object):
                         'count': 1,
                         'date': record['date']
                     }
-            if record['record_type'] in ['bpd', 'bpi']:
+            if record['record_type'] in ['bpd', 'bpc']:
                 valid_blueprints[record['id']] = {
                     'primary_key': record['primary_key'],
                     'count': 0,
@@ -401,12 +401,13 @@ class RecordProcessor(object):
                 else:
                     users_reviews[launchpad_id] = [review]
 
-        for bp in valid_blueprints.keys():
+        for bp_name, bp in valid_blueprints.iteritems():
-            if bp in mentioned_blueprints:
+            if bp_name in mentioned_blueprints:
-                valid_blueprints[bp]['count'] = (
+                bp['count'] = mentioned_blueprints[bp_name]['count']
-                    mentioned_blueprints[bp]['count'])
+                bp['date'] = mentioned_blueprints[bp_name]['date']
-                valid_blueprints[bp]['date'] = (
+            else:
-                    mentioned_blueprints[bp]['date'])
+                bp['count'] = 0
+                bp['date'] = 0
 
         reviews_index = {}
         for launchpad_id, reviews in users_reviews.iteritems():
@@ -441,7 +442,7 @@ class RecordProcessor(object):
                     need_update = True
             record['blueprint_id'] = list(valid_bp)
 
-            if record['record_type'] in ['bpd', 'bpi']:
+            if record['record_type'] in ['bpd', 'bpc']:
                 bp = valid_blueprints[record['id']]
                 if ((record.get('mention_count') != bp['count']) or
                         (record.get('mention_date') != bp['date'])):