x/stackalytics
Code Issues Proposed changes

Implemented stats for external CIs

Browse Source 
All CI votes are collected in new records with type 'ci_vote'. All standard statistics are
available for these records. Also added a new "CI Status" report that shows summary of CI
tests execution on merged patch sets, overall summary and results per merged change requests.

List of CIs is taken from DriverLog's repo.

Change-Id: Ic7a830dc5b331ba4c099be458ad2bab4c2072607
This commit is contained in:
Ilya Shakhat 2014-06-29 13:31:30 +04:00
parent 1865fc804f
commit 5ad1cbe79c
14 changed files with 493 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
etc/stackalytics.conf
View File

@ -41,6 +41,9 @@
# The address of file with list of programs
# program_list_uri = https://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml
# The address of DriverLog data
# driverlog_data_uri = https://git.openstack.org/cgit/stackforge/driverlog/plain/etc/default_data.json
# Default metric
# default_metric = marks

1
stackalytics/dashboard/decorators.py
View File

@ -314,6 +314,7 @@ def aggregate_filter():
'resolved-bugs': (incremental_filter, None),
'members': (incremental_filter, None),
'person-day': (person_day_filter, None),
'ci': (None, None),
}
if metric not in metric_to_filters_map:
metric = parameters.get_default('metric')

4
stackalytics/dashboard/helpers.py
View File

@ -14,6 +14,7 @@
# limitations under the License.
import datetime
import operator
import re
import six
@ -116,7 +117,8 @@ def get_activity(records, start_record, page_size, query_message=None):
records = [r for r in records
if (r.get('message') and
r.get('message').find(query_message) > 0)]
records_sorted = sorted(records, key=lambda x: x['date'], reverse=True)
records_sorted = sorted(records, key=operator.itemgetter('date'),
reverse=True)
result = []
for record in records_sorted[start_record:]:

2
stackalytics/dashboard/parameters.py
View File

@ -38,6 +38,7 @@ METRIC_LABELS = {
'filed-bugs': 'Filed Bugs',
'resolved-bugs': 'Resolved Bugs',
# 'person-day': "Person-day effort"
'ci': 'CI votes',
}
METRIC_TO_RECORD_TYPE = {
@ -50,6 +51,7 @@ METRIC_TO_RECORD_TYPE = {
'filed-bugs': 'bugf',
'resolved-bugs': 'bugr',
'members': 'member',
'ci': 'ci_vote',
}
FILTER_PARAMETERS = ['release', 'project_type', 'module', 'company', 'user_id',

16
stackalytics/dashboard/reports.py
View File

@ -129,6 +129,20 @@ def contribution(module, days):
}
@blueprint.route('/ci/<module>/<days>')
@decorators.templated()
@decorators.exception_handler()
def external_ci(module, days):
if int(days) > 30:
days = 30
return {
'module': module,
'days': days,
'start_date': int(time.time()) - int(days) * 24 * 60 * 60
}
@blueprint.route('/members')
@decorators.exception_handler()
@decorators.templated()
@ -170,7 +184,7 @@ def _get_punch_card_data(records):
def _get_activity_summary(record_ids):
memory_storage_inst = vault.get_memory_storage()
types = ['mark', 'patch', 'email', 'bpd', 'bpc']
types = ['mark', 'patch', 'email', 'bpd', 'bpc', 'ci_vote']
record_ids_by_type = set()
for t in types:
record_ids_by_type |= memory_storage_inst.get_record_ids_by_type(t)

6
stackalytics/dashboard/templates/_macros/activity_log.html
View File

@ -149,6 +149,12 @@ show_record_type=True, show_user_gravatar=True, gravatar_size=32, show_all=True)
<div class="header">&ldquo;${title}&rdquo;</div>
<div>Status: <span class="status${status}">${status}</span></div>
<div>Importance: ${importance}</div>
{%elif record_type == "ci_vote" %}
<div class="header">New CI vote in change request ${review_number}
{%if is_merged %}(<span style="color: green;">Merged</span>){%/if%}</div>
<div>Parsed result: {%if ci_result == true %}<span style="color: green">Success</span>{%else%}<span style="color: red">Failure</span>{%/if%}</div>
<div>Message: ${message}</div>
<div>Change Id: <a href="https://review.openstack.org/#/c/${review_number}" target="_blank">${review_id}</a></div>
{%/if%}
</div>
</div>

3
stackalytics/dashboard/templates/overview.html
View File

@ -21,6 +21,9 @@
<div><b><a href="/report/reviews/{{ module }}/open" target="_blank">Show open reviews for {{ module_inst.module_group_name }}</a></b></div>
<div><b><a href="/report/contribution/{{ module }}/30" target="_blank">Contribution for the last 30 days in {{ module_inst.module_group_name }}</a></b></div>
<div><b><a href="/report/contribution/{{ module }}/90" target="_blank">Contribution for the last 90 days in {{ module_inst.module_group_name }}</a></b></div>
{% if module_inst.ci %}
<div><b><a href="/report/ci/{{ module }}/7" target="_blank">External CI status for {{ module_inst.module_group_name }} for the last 7 days</a></b></div>
{% endif %}
{% endif %}
{% if company %}
<div><b><a href="/report/companies/{{ company }}" target="_blank">Show activity report for {{ company_original }}</a></b></div>

272
stackalytics/dashboard/templates/reports/external_ci.html Normal file
View File

@ -0,0 +1,272 @@
{% extends "reports/base_report.html" %}
{% block title %}
External CI status for {{ module }} for the last {{ days }} days
{% endblock %}
{% block scripts %}
<script type="text/javascript">
jQuery.extend(jQuery.fn.dataTableExt.oSort, {
"ratio-pre": function (a) {
if (a == "&#x221E;" || a == "-")
return -1.0;
a = a.substr(0, a.length - 1);
return parseFloat(a);
},
"ratio-asc": function (a, b) {
return ((a < b) ? -1 : ((a > b) ? 1 : 0));
},
"ratio-desc": function (a, b) {
return ((a < b) ? 1 : ((a > b) ? -1 : 0));
}
});
$(document).ready(function () {
var table_column_names = ["name", "merge_run_count", "merge_success_rate", "merge_date_str", "merge_result",
"total_run_count", "total_success_rate", "total_date_str"];
var table_id = "summary_stats_table";
$.ajax({
url: makeURI("/api/1.0/activity?project_type=all&metric=ci&module={{ module }}&release=all&start_date={{ start_date }}&page_size=-1"),
dataType: "json",
success: function (data) {
var activity = data["activity"];
var tableColumns = [];
for (var i = 0; i < table_column_names.length; i++) {
tableColumns.push({"mData": table_column_names[i]});
}
var ci_map = {}; // ci_id -> statistics
var res_map = {}; // ci_id -> map: review_id -> res
var ci_id_to_name = {}; // ci_id -> name
var review_ids = [];
var review_ids_set = {};
for (i = 0; i < activity.length; i++) {
var ai = activity[i];
if (ai.branch != "master") {
continue;
}
var ci_id = ai.user_id;
var ci = ci_map[ci_id];
if (!ci) {
ci = {total_run_count: 0, total_success: 0, merge_run_count: 0, merge_success: 0,
merge_date: 0, total_date: 0, merge_result: "N/A",
merge_date_str: "N/A", "total_date_str": "N/A"};
}
ci.id = ai.user_id;
ci.name = "<a href=\"https://review.openstack.org/#/q/reviewer:" + ai.user_id +
"+project:openstack/{{ module }},n,z\" target=\"_blank\">" + ai.user_id + "</a>";
ci_id_to_name[ci_id] = ci.name;
ci.total_run_count ++;
if (ai.ci_result) {
ci.total_success ++;
}
if (ai.is_merged) {
var review_id = ai.review_number;
if (!review_ids_set[review_id]) {
review_ids_set[review_id] = review_id;
review_ids.push({review_id: review_id, date_str: ai.date_str});
}
if (!res_map[ci_id]) {
res_map[ci_id] = {};
}
res_map[ci_id][review_id] = ai.ci_result;
ci.merge_run_count ++;
if (ai.ci_result) {
ci.merge_success ++;
}
if (ai.date > ci.merge_date) {
ci.merge_date = ai.date;
ci.merge_date_str = ai.date_str;
ci.merge_result = (ai.ci_result)? "<span style=\"color: green\">&#x2714;</span>":
"<span style=\"color: red\">&#x2716;</span>";
}
}
if (ai.date > ci.total_date) {
ci.total_date = ai.date;
ci.total_date_str = ai.date_str;
}
ci_map[ci_id] = ci;
}
var tableData = [];
for (ci_id in ci_map) {
var c = ci_map[ci_id];
if (c.merge_run_count > 0) {
c.merge_success_rate = Math.round(c.merge_success / c.merge_run_count * 100) + "%";
} else {
c.merge_success_rate = "-";
}
if (c.total_run_count > 0) {
c.total_success_rate = Math.round(c.total_success / c.total_run_count * 100) + "%";
} else {
c.total_success_rate = "-";
}
tableData.push(c);
}
$("#" + table_id).dataTable({
"aaSorting": [
[ 0, "asc"]
],
"bFilter": true,
"bInfo": false,
"bAutoWidth": false,
"bPaginate": false,
"iDisplayLength": -1,
"aaData": tableData,
"aoColumns": tableColumns,
"aoColumnDefs": [
{ "sClass": "center", "aTargets": [1, 2, 3, 4, 5, 6, 7] },
{ "sType": "ratio", "aTargets": [2, 6]}
]
}).show();
$("#" + table_id + "_loading").hide();
// make table with per-change-request status
var table = $("<table id='table_status'></table>");
var table_head = $('<thead></thead>');
table.append(table_head);
var table_head_row = $("<tr></tr>");
table_head.append(table_head_row);
table_head_row.append($("<th rowspan='2'>CI</th>"));
var table_second_head_row = $("<tr></tr>");
table_head.append(table_second_head_row);
var table_body = $("<tbody></tbody>");
table.append(table_body);
var prev_date = null;
var count = 0;
for (i = review_ids.length - 1; i >=0; i--) {
var date_str = review_ids[i].date_str;
var dp = date_str.split(" ");
var date = dp[0] + " " + dp[1];
if (date != prev_date && count > 0) {
table_head_row.append($("<th colspan='" + count + "'>" + prev_date + "</th>"));
prev_date = date;
count = 0;
}
prev_date = date;
count++;
table_second_head_row.append($("<th title='" + review_ids[i].review_id + "'></th>"));
}
if (count > 0) {
table_head_row.append($("<th colspan='" + count + "'>" + prev_date + "</th>"));
}
for (ci_id in ci_id_to_name) {
var table_row = $("<tr></tr>");
table_row.append($("<td>" + ci_id_to_name[ci_id] + "</td>"));
for (i = review_ids.length - 1; i >=0; i--) {
var review_id = review_ids[i].review_id;
var color = "#cfcfcf";
if (res_map[ci_id]) {
var res = res_map[ci_id][review_id];
if (res == true) {
color = "#40bf00";
} else if (res == false) {
color = "#bf4000";
}
}
const url = "https://review.openstack.org/#/c/" + review_id;
var cell = $("<td></td>");
cell.css("background-color", color).css("cursor", "pointer").prop("title", url);
cell.click(function(evt) {
window.open($(this).prop("title"));
});
table_row.append(cell);
}
table_body.append(table_row);
}
$("#table_status_container").append(table);
$("#table_status_container_loading").hide();
table.dataTable({
"bPaginate": false,
"iDisplayLength": -1
});
}
});
});
</script>
<style type="text/css">
table.dataTable tr.even {
background-color: #EEF1F4;
}
table.dataTable tr.even:hover, table.dataTable tr.odd:hover {
background-color: #F8FFEC;
}
table.dataTable tr.even td.sorting_1 {
background-color: #E0E8E8;
}
</style>
{% endblock %}
{% block content %}
<h1>External CI status for {{ module }} for the last {{ days }} days</h1>
<h2>Summary</h2>
<p>Overall stats for external CIs. <i>Merged patch sets</i> - results of test execution on the last patch set in the
merged change request (assuming that it is almost the same as to run tests against master). <i>All patch sets</i> -
results of test execution on all patch sets.</p>
<p>List of CIs is taken from <a href="http://localhost:5000/report/driverlog">DriverLog</a> and can be updated by commit into its
<a href="https://git.openstack.org/cgit/stackforge/driverlog/tree/etc/default_data.json">default_data.json</a></p>
<table id="summary_stats_table" style="display: none;">
<thead>
<tr>
<th rowspan="2">CI</th>
<th colspan="4" title="Stats for the last patch set in the merged change request">Merged patch sets</th>
<th colspan="3">All patch sets</th>
</tr>
<tr>
<th>Total runs</th>
<th>Success, %</th>
<th>Last run</th>
<th>Last result</th>
<th>Total runs</th>
<th>Success, %</th>
<th>Last run</th>
</tr>
</thead>
<tbody>
</tbody>
</table>
<div id="summary_stats_table_loading" class="select2-loading" style="width: 7em;">Loading...</div>
<h2>Status for merged changed requests</h2>
<p>Status of CI test execution for every merged patch set grouped by days. Green cell - tests ran successfully,
red cell - tests failed, grey cell - tests results not found.</p>
<div id="table_status_container"></div>
<div id="table_status_container_loading" class="select2-loading" style="width: 7em;">Loading...</div>
{% endblock %}

4
stackalytics/processor/config.py
View File

@ -49,6 +49,10 @@ OPTS = [
default=('https://git.openstack.org/cgit/'
'openstack/governance/plain/reference/programs.yaml'),
help='The address of file with list of programs'),
cfg.StrOpt('driverlog-data-uri',
default='https://git.openstack.org/cgit/'
'stackforge/driverlog/plain/etc/default_data.json',
help='URI for default data'),
cfg.StrOpt('default-metric', default='marks',
help='Default metric'),
cfg.StrOpt('default-release',

29
stackalytics/processor/default_data_processor.py
View File

@ -109,6 +109,31 @@ def _update_project_list(default_data, git_base_uri, gerrit):
default_data['project_sources'], default_data['repos'])
def _update_with_driverlog_data(default_data, driverlog_data_uri):
LOG.info('Reading DriverLog data from uri: %s', driverlog_data_uri)
driverlog_data = utils.read_json_from_uri(driverlog_data_uri)
cis = {}
for driver in driverlog_data['drivers']:
if 'ci' in driver:
module = driver['project_id'].split('/')[1]
if module not in cis:
cis[module] = {}
cis[module][driver['ci']['id']] = driver
default_data['users'].append({
'launchpad_id': driver['ci']['id'],
'user_name': driver['ci']['id'],
'companies': [
{'company_name': driver['vendor'], 'end_date': None}],
})
for repo in default_data['repos']:
if repo['module'] in cis:
repo['ci'] = cis[repo['module']]
def _store_users(runtime_storage_inst, users):
for user in users:
stored_user = utils.load_user(runtime_storage_inst, user['user_id'])
@ -165,10 +190,12 @@ def _store_default_data(runtime_storage_inst, default_data):
def process(runtime_storage_inst, default_data,
git_base_uri, gerrit):
git_base_uri, gerrit, driverlog_data_uri):
LOG.debug('Process default data')
if 'project_sources' in default_data:
_update_project_list(default_data, git_base_uri, gerrit)
_update_with_driverlog_data(default_data, driverlog_data_uri)
_store_default_data(runtime_storage_inst, default_data)

92
stackalytics/processor/driverlog.py Normal file
View File

@ -0,0 +1,92 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import re
from stackalytics.openstack.common import log as logging
LOG = logging.getLogger(__name__)
def _find_vote(review, ci_id, patch_set_number):
"""
Finds vote corresponding to ci_id
"""
for patch_set in review['patchSets']:
if patch_set['number'] == patch_set_number:
for approval in (patch_set.get('approvals') or []):
if approval['type'] not in ['Verified', 'VRIF']:
continue
if approval['by'].get('username') == ci_id:
return approval['value'] in ['1', '2']
return None
def find_ci_result(review, ci_map):
"""
For a given stream of reviews yields results produced by CIs
"""
review_id = review['id']
review_number = review['number']
for comment in reversed(review.get('comments') or []):
reviewer_id = comment['reviewer'].get('username')
if reviewer_id not in ci_map:
continue
message = comment['message']
m = re.match(r'Patch Set (?P<number>\d+):(?P<message>.*)',
message, flags=re.DOTALL)
if not m:
continue # do not understand comment
patch_set_number = m.groupdict()['number']
message = m.groupdict()['message'].strip()
result = None
ci = ci_map[reviewer_id]['ci']
# try to get result by parsing comment message
success_pattern = ci.get('success_pattern')
failure_pattern = ci.get('failure_pattern')
if success_pattern and re.search(success_pattern, message):
result = True
elif failure_pattern and re.search(failure_pattern, message):
result = False
# try to get result from vote
if result is None:
result = _find_vote(review, ci['id'], patch_set_number)
if result is not None:
is_merged = (review['status'] == 'MERGED' and
patch_set_number == review['patchSets'][-1]
['number'])
yield {
'reviewer': comment['reviewer'],
'ci_result': result,
'is_merged': is_merged,
'message': message,
'date': comment['timestamp'],
'review_id': review_id,
'review_number': review_number,
'driver_name': ci_map[reviewer_id]['name'],
'driver_vendor': ci_map[reviewer_id]['vendor'],
}

28
stackalytics/processor/main.py
View File

@ -25,6 +25,7 @@ from stackalytics.openstack.common import log as logging
from stackalytics.processor import bps
from stackalytics.processor import config
from stackalytics.processor import default_data_processor
from stackalytics.processor import driverlog
from stackalytics.processor import lp
from stackalytics.processor import mls
from stackalytics.processor import mps
@ -77,6 +78,22 @@ def _record_typer(record_iterator, record_type):
yield record
def _process_reviews(record_iterator, ci_map, module, branch):
for record in record_iterator:
yield record
for driver_info in driverlog.find_ci_result(record, ci_map):
driver_info['record_type'] = 'ci_vote'
driver_info['module'] = module
driver_info['branch'] = branch
release = branch.lower()
if release.find('/') > 0:
driver_info['release'] = release.split('/')[1]
yield driver_info
def _process_repo(repo, runtime_storage_inst, record_processor_inst,
bug_modified_since):
uri = repo['uri']
@ -131,8 +148,14 @@ def _process_repo(repo, runtime_storage_inst, record_processor_inst,
rcs_key = 'rcs:' + str(parse.quote_plus(uri) + ':' + branch)
last_id = runtime_storage_inst.get_by_key(rcs_key)
review_iterator = rcs_inst.log(branch, last_id)
review_iterator = rcs_inst.log(branch, last_id,
grab_comments=('ci' in repo))
review_iterator_typed = _record_typer(review_iterator, 'review')
if 'ci' in repo: # add external CI data
review_iterator_typed = _process_reviews(
review_iterator_typed, repo['ci'], repo['module'], branch)
processed_review_iterator = record_processor_inst.process(
review_iterator_typed)
runtime_storage_inst.set_records(processed_review_iterator,
@ -310,7 +333,8 @@ def main():
default_data_processor.process(runtime_storage_inst,
default_data,
cfg.CONF.git_base_uri,
gerrit)
gerrit,
cfg.CONF.driverlog_data_uri)
process_program_list(runtime_storage_inst, cfg.CONF.program_list_uri)

17
stackalytics/processor/rcs.py
View File

@ -84,7 +84,7 @@ class Gerrit(Rcs):
return False
def _get_cmd(self, project_organization, module, branch, sort_key=None,
is_open=False, limit=PAGE_LIMIT):
is_open=False, limit=PAGE_LIMIT, grab_comments=False):
cmd = ('gerrit query --all-approvals --patch-sets --format JSON '
'project:\'%(ogn)s/%(module)s\' branch:%(branch)s '
'limit:%(limit)s' %
@ -94,6 +94,8 @@ class Gerrit(Rcs):
cmd += ' is:open'
if sort_key:
cmd += ' resume_sortkey:%016x' % sort_key
if grab_comments:
cmd += ' --comments'
return cmd
def _exec_command(self, cmd):
@ -106,12 +108,13 @@ class Gerrit(Rcs):
return False
def _poll_reviews(self, project_organization, module, branch,
start_id=None, last_id=None, is_open=False):
start_id=None, last_id=None, is_open=False,
grab_comments=False):
sort_key = start_id
while True:
cmd = self._get_cmd(project_organization, module, branch, sort_key,
is_open)
is_open, grab_comments=grab_comments)
LOG.debug('Executing command: %s', cmd)
exec_result = self._exec_command(cmd)
if not exec_result:
@ -148,7 +151,7 @@ class Gerrit(Rcs):
return result
def log(self, branch, last_id):
def log(self, branch, last_id, grab_comments=False):
if not self._connect():
return
@ -156,7 +159,8 @@ class Gerrit(Rcs):
LOG.debug('Poll new reviews for module: %s', self.repo['module'])
for review in self._poll_reviews(self.repo['organization'],
self.repo['module'], branch,
last_id=last_id):
last_id=last_id,
grab_comments=grab_comments):
yield review
# poll open reviews from last_id down to bottom
@ -166,7 +170,8 @@ class Gerrit(Rcs):
start_id = last_id + 1 # include the last review into query
for review in self._poll_reviews(self.repo['organization'],
self.repo['module'], branch,
start_id=start_id, is_open=True):
start_id=start_id, is_open=True,
grab_comments=grab_comments):
yield review
self.client.close()

52
stackalytics/processor/record_processor.py
View File

@ -473,25 +473,22 @@ class RecordProcessor(object):
yield record
def _apply_type_based_processing(self, record):
if record['record_type'] == 'commit':
for r in self._process_commit(record):
yield r
elif record['record_type'] == 'review':
for r in self._process_review(record):
yield r
elif record['record_type'] == 'email':
for r in self._process_email(record):
yield r
elif record['record_type'] == 'bp':
for r in self._process_blueprint(record):
yield r
elif record['record_type'] == 'member':
for r in self._process_member(record):
yield r
elif record['record_type'] == 'bug':
for r in self._process_bug(record):
yield r
def _process_ci(self, record):
ci_vote = dict((k, v) for k, v in six.iteritems(record)
if k not in ['reviewer'])
reviewer = record['reviewer']
ci_vote['primary_key'] = ('%s:%s' % (reviewer['username'],
ci_vote['date']))
ci_vote['user_id'] = reviewer['username']
ci_vote['launchpad_id'] = reviewer['username']
ci_vote['author_name'] = reviewer.get('name') or reviewer['username']
ci_vote['author_email'] = (
reviewer.get('email') or reviewer['username']).lower()
self._update_record_and_user(ci_vote)
yield ci_vote
def _renew_record_date(self, record):
record['week'] = utils.timestamp_to_week(record['date'])
@ -499,14 +496,19 @@ class RecordProcessor(object):
record['release'] = self._get_release(record['date'])
def process(self, record_iterator):
PROCESSORS = {
'commit': self._process_commit,
'review': self._process_review,
'email': self._process_email,
'bp': self._process_blueprint,
'bug': self._process_bug,
'member': self._process_member,
'ci_vote': self._process_ci,
}
for record in record_iterator:
for r in self._apply_type_based_processing(record):
if r['company_name'] == '*robots':
continue
for r in PROCESSORS[record['record_type']](record):
self._renew_record_date(r)
yield r
def _update_records_with_releases(self, release_index):