diff --git a/etc/stackalytics.conf b/etc/stackalytics.conf index 9ab56c097..909a7a66a 100644 --- a/etc/stackalytics.conf +++ b/etc/stackalytics.conf @@ -41,6 +41,9 @@ # The address of file with list of programs # program_list_uri = https://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml +# The address of DriverLog data +# driverlog_data_uri = https://git.openstack.org/cgit/stackforge/driverlog/plain/etc/default_data.json + # Default metric # default_metric = marks diff --git a/stackalytics/dashboard/decorators.py b/stackalytics/dashboard/decorators.py index 775000c52..feb58487d 100644 --- a/stackalytics/dashboard/decorators.py +++ b/stackalytics/dashboard/decorators.py @@ -314,6 +314,7 @@ def aggregate_filter(): 'resolved-bugs': (incremental_filter, None), 'members': (incremental_filter, None), 'person-day': (person_day_filter, None), + 'ci': (None, None), } if metric not in metric_to_filters_map: metric = parameters.get_default('metric') diff --git a/stackalytics/dashboard/helpers.py b/stackalytics/dashboard/helpers.py index aae99d814..7dcefdd91 100644 --- a/stackalytics/dashboard/helpers.py +++ b/stackalytics/dashboard/helpers.py @@ -14,6 +14,7 @@ # limitations under the License. import datetime +import operator import re import six @@ -116,7 +117,8 @@ def get_activity(records, start_record, page_size, query_message=None): records = [r for r in records if (r.get('message') and r.get('message').find(query_message) > 0)] - records_sorted = sorted(records, key=lambda x: x['date'], reverse=True) + records_sorted = sorted(records, key=operator.itemgetter('date'), + reverse=True) result = [] for record in records_sorted[start_record:]: diff --git a/stackalytics/dashboard/parameters.py b/stackalytics/dashboard/parameters.py index 944fc5f9d..2e93fbed4 100644 --- a/stackalytics/dashboard/parameters.py +++ b/stackalytics/dashboard/parameters.py @@ -38,6 +38,7 @@ METRIC_LABELS = { 'filed-bugs': 'Filed Bugs', 'resolved-bugs': 'Resolved Bugs', # 'person-day': "Person-day effort" + 'ci': 'CI votes', } METRIC_TO_RECORD_TYPE = { @@ -50,6 +51,7 @@ METRIC_TO_RECORD_TYPE = { 'filed-bugs': 'bugf', 'resolved-bugs': 'bugr', 'members': 'member', + 'ci': 'ci_vote', } FILTER_PARAMETERS = ['release', 'project_type', 'module', 'company', 'user_id', diff --git a/stackalytics/dashboard/reports.py b/stackalytics/dashboard/reports.py index 3958fc96c..d9efc7290 100644 --- a/stackalytics/dashboard/reports.py +++ b/stackalytics/dashboard/reports.py @@ -129,6 +129,20 @@ def contribution(module, days): } +@blueprint.route('/ci//') +@decorators.templated() +@decorators.exception_handler() +def external_ci(module, days): + if int(days) > 30: + days = 30 + + return { + 'module': module, + 'days': days, + 'start_date': int(time.time()) - int(days) * 24 * 60 * 60 + } + + @blueprint.route('/members') @decorators.exception_handler() @decorators.templated() @@ -170,7 +184,7 @@ def _get_punch_card_data(records): def _get_activity_summary(record_ids): memory_storage_inst = vault.get_memory_storage() - types = ['mark', 'patch', 'email', 'bpd', 'bpc'] + types = ['mark', 'patch', 'email', 'bpd', 'bpc', 'ci_vote'] record_ids_by_type = set() for t in types: record_ids_by_type |= memory_storage_inst.get_record_ids_by_type(t) diff --git a/stackalytics/dashboard/templates/_macros/activity_log.html b/stackalytics/dashboard/templates/_macros/activity_log.html index 25db73731..1d440c622 100644 --- a/stackalytics/dashboard/templates/_macros/activity_log.html +++ b/stackalytics/dashboard/templates/_macros/activity_log.html @@ -149,6 +149,12 @@ show_record_type=True, show_user_gravatar=True, gravatar_size=32, show_all=True)
“${title}”
Status: ${status}
Importance: ${importance}
+ {%elif record_type == "ci_vote" %} +
New CI vote in change request ${review_number} + {%if is_merged %}(Merged){%/if%}
+
Parsed result: {%if ci_result == true %}Success{%else%}Failure{%/if%}
+
Message: ${message}
+
Change Id: ${review_id}
{%/if%} diff --git a/stackalytics/dashboard/templates/overview.html b/stackalytics/dashboard/templates/overview.html index e80976ac7..9242950c4 100644 --- a/stackalytics/dashboard/templates/overview.html +++ b/stackalytics/dashboard/templates/overview.html @@ -21,6 +21,9 @@
Show open reviews for {{ module_inst.module_group_name }}
Contribution for the last 30 days in {{ module_inst.module_group_name }}
Contribution for the last 90 days in {{ module_inst.module_group_name }}
+ {% if module_inst.ci %} +
External CI status for {{ module_inst.module_group_name }} for the last 7 days
+ {% endif %} {% endif %} {% if company %}
Show activity report for {{ company_original }}
diff --git a/stackalytics/dashboard/templates/reports/external_ci.html b/stackalytics/dashboard/templates/reports/external_ci.html new file mode 100644 index 000000000..33e43ea17 --- /dev/null +++ b/stackalytics/dashboard/templates/reports/external_ci.html @@ -0,0 +1,272 @@ +{% extends "reports/base_report.html" %} + +{% block title %} +External CI status for {{ module }} for the last {{ days }} days +{% endblock %} + +{% block scripts %} + + + + +{% endblock %} + +{% block content %} +

External CI status for {{ module }} for the last {{ days }} days

+ +

Summary

+ +

Overall stats for external CIs. Merged patch sets - results of test execution on the last patch set in the + merged change request (assuming that it is almost the same as to run tests against master). All patch sets - + results of test execution on all patch sets.

+

List of CIs is taken from DriverLog and can be updated by commit into its + default_data.json

+ + + + + + + + + + + + + + + + + + + + + + +
Loading...
+ +

Status for merged changed requests

+ +

Status of CI test execution for every merged patch set grouped by days. Green cell - tests ran successfully, + red cell - tests failed, grey cell - tests results not found.

+ +
+
Loading...
+ +{% endblock %} diff --git a/stackalytics/processor/config.py b/stackalytics/processor/config.py index 63fce1703..f9ae5360c 100644 --- a/stackalytics/processor/config.py +++ b/stackalytics/processor/config.py @@ -49,6 +49,10 @@ OPTS = [ default=('https://git.openstack.org/cgit/' 'openstack/governance/plain/reference/programs.yaml'), help='The address of file with list of programs'), + cfg.StrOpt('driverlog-data-uri', + default='https://git.openstack.org/cgit/' + 'stackforge/driverlog/plain/etc/default_data.json', + help='URI for default data'), cfg.StrOpt('default-metric', default='marks', help='Default metric'), cfg.StrOpt('default-release', diff --git a/stackalytics/processor/default_data_processor.py b/stackalytics/processor/default_data_processor.py index 5bb681312..5dd08ce01 100644 --- a/stackalytics/processor/default_data_processor.py +++ b/stackalytics/processor/default_data_processor.py @@ -109,6 +109,31 @@ def _update_project_list(default_data, git_base_uri, gerrit): default_data['project_sources'], default_data['repos']) +def _update_with_driverlog_data(default_data, driverlog_data_uri): + LOG.info('Reading DriverLog data from uri: %s', driverlog_data_uri) + driverlog_data = utils.read_json_from_uri(driverlog_data_uri) + + cis = {} + for driver in driverlog_data['drivers']: + if 'ci' in driver: + module = driver['project_id'].split('/')[1] + + if module not in cis: + cis[module] = {} + cis[module][driver['ci']['id']] = driver + + default_data['users'].append({ + 'launchpad_id': driver['ci']['id'], + 'user_name': driver['ci']['id'], + 'companies': [ + {'company_name': driver['vendor'], 'end_date': None}], + }) + + for repo in default_data['repos']: + if repo['module'] in cis: + repo['ci'] = cis[repo['module']] + + def _store_users(runtime_storage_inst, users): for user in users: stored_user = utils.load_user(runtime_storage_inst, user['user_id']) @@ -165,10 +190,12 @@ def _store_default_data(runtime_storage_inst, default_data): def process(runtime_storage_inst, default_data, - git_base_uri, gerrit): + git_base_uri, gerrit, driverlog_data_uri): LOG.debug('Process default data') if 'project_sources' in default_data: _update_project_list(default_data, git_base_uri, gerrit) + _update_with_driverlog_data(default_data, driverlog_data_uri) + _store_default_data(runtime_storage_inst, default_data) diff --git a/stackalytics/processor/driverlog.py b/stackalytics/processor/driverlog.py new file mode 100644 index 000000000..e5459ff55 --- /dev/null +++ b/stackalytics/processor/driverlog.py @@ -0,0 +1,92 @@ +# Copyright (c) 2014 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +from stackalytics.openstack.common import log as logging + + +LOG = logging.getLogger(__name__) + + +def _find_vote(review, ci_id, patch_set_number): + """ + Finds vote corresponding to ci_id + """ + for patch_set in review['patchSets']: + if patch_set['number'] == patch_set_number: + for approval in (patch_set.get('approvals') or []): + if approval['type'] not in ['Verified', 'VRIF']: + continue + + if approval['by'].get('username') == ci_id: + return approval['value'] in ['1', '2'] + + return None + + +def find_ci_result(review, ci_map): + """ + For a given stream of reviews yields results produced by CIs + """ + + review_id = review['id'] + review_number = review['number'] + + for comment in reversed(review.get('comments') or []): + reviewer_id = comment['reviewer'].get('username') + if reviewer_id not in ci_map: + continue + + message = comment['message'] + m = re.match(r'Patch Set (?P\d+):(?P.*)', + message, flags=re.DOTALL) + if not m: + continue # do not understand comment + + patch_set_number = m.groupdict()['number'] + message = m.groupdict()['message'].strip() + + result = None + ci = ci_map[reviewer_id]['ci'] + + # try to get result by parsing comment message + success_pattern = ci.get('success_pattern') + failure_pattern = ci.get('failure_pattern') + + if success_pattern and re.search(success_pattern, message): + result = True + elif failure_pattern and re.search(failure_pattern, message): + result = False + + # try to get result from vote + if result is None: + result = _find_vote(review, ci['id'], patch_set_number) + + if result is not None: + is_merged = (review['status'] == 'MERGED' and + patch_set_number == review['patchSets'][-1] + ['number']) + yield { + 'reviewer': comment['reviewer'], + 'ci_result': result, + 'is_merged': is_merged, + 'message': message, + 'date': comment['timestamp'], + 'review_id': review_id, + 'review_number': review_number, + 'driver_name': ci_map[reviewer_id]['name'], + 'driver_vendor': ci_map[reviewer_id]['vendor'], + } diff --git a/stackalytics/processor/main.py b/stackalytics/processor/main.py index 35331b313..b1773b461 100644 --- a/stackalytics/processor/main.py +++ b/stackalytics/processor/main.py @@ -25,6 +25,7 @@ from stackalytics.openstack.common import log as logging from stackalytics.processor import bps from stackalytics.processor import config from stackalytics.processor import default_data_processor +from stackalytics.processor import driverlog from stackalytics.processor import lp from stackalytics.processor import mls from stackalytics.processor import mps @@ -77,6 +78,22 @@ def _record_typer(record_iterator, record_type): yield record +def _process_reviews(record_iterator, ci_map, module, branch): + for record in record_iterator: + yield record + + for driver_info in driverlog.find_ci_result(record, ci_map): + driver_info['record_type'] = 'ci_vote' + driver_info['module'] = module + driver_info['branch'] = branch + + release = branch.lower() + if release.find('/') > 0: + driver_info['release'] = release.split('/')[1] + + yield driver_info + + def _process_repo(repo, runtime_storage_inst, record_processor_inst, bug_modified_since): uri = repo['uri'] @@ -131,8 +148,14 @@ def _process_repo(repo, runtime_storage_inst, record_processor_inst, rcs_key = 'rcs:' + str(parse.quote_plus(uri) + ':' + branch) last_id = runtime_storage_inst.get_by_key(rcs_key) - review_iterator = rcs_inst.log(branch, last_id) + review_iterator = rcs_inst.log(branch, last_id, + grab_comments=('ci' in repo)) review_iterator_typed = _record_typer(review_iterator, 'review') + + if 'ci' in repo: # add external CI data + review_iterator_typed = _process_reviews( + review_iterator_typed, repo['ci'], repo['module'], branch) + processed_review_iterator = record_processor_inst.process( review_iterator_typed) runtime_storage_inst.set_records(processed_review_iterator, @@ -310,7 +333,8 @@ def main(): default_data_processor.process(runtime_storage_inst, default_data, cfg.CONF.git_base_uri, - gerrit) + gerrit, + cfg.CONF.driverlog_data_uri) process_program_list(runtime_storage_inst, cfg.CONF.program_list_uri) diff --git a/stackalytics/processor/rcs.py b/stackalytics/processor/rcs.py index f57d22f40..86b1aba93 100644 --- a/stackalytics/processor/rcs.py +++ b/stackalytics/processor/rcs.py @@ -84,7 +84,7 @@ class Gerrit(Rcs): return False def _get_cmd(self, project_organization, module, branch, sort_key=None, - is_open=False, limit=PAGE_LIMIT): + is_open=False, limit=PAGE_LIMIT, grab_comments=False): cmd = ('gerrit query --all-approvals --patch-sets --format JSON ' 'project:\'%(ogn)s/%(module)s\' branch:%(branch)s ' 'limit:%(limit)s' % @@ -94,6 +94,8 @@ class Gerrit(Rcs): cmd += ' is:open' if sort_key: cmd += ' resume_sortkey:%016x' % sort_key + if grab_comments: + cmd += ' --comments' return cmd def _exec_command(self, cmd): @@ -106,12 +108,13 @@ class Gerrit(Rcs): return False def _poll_reviews(self, project_organization, module, branch, - start_id=None, last_id=None, is_open=False): + start_id=None, last_id=None, is_open=False, + grab_comments=False): sort_key = start_id while True: cmd = self._get_cmd(project_organization, module, branch, sort_key, - is_open) + is_open, grab_comments=grab_comments) LOG.debug('Executing command: %s', cmd) exec_result = self._exec_command(cmd) if not exec_result: @@ -148,7 +151,7 @@ class Gerrit(Rcs): return result - def log(self, branch, last_id): + def log(self, branch, last_id, grab_comments=False): if not self._connect(): return @@ -156,7 +159,8 @@ class Gerrit(Rcs): LOG.debug('Poll new reviews for module: %s', self.repo['module']) for review in self._poll_reviews(self.repo['organization'], self.repo['module'], branch, - last_id=last_id): + last_id=last_id, + grab_comments=grab_comments): yield review # poll open reviews from last_id down to bottom @@ -166,7 +170,8 @@ class Gerrit(Rcs): start_id = last_id + 1 # include the last review into query for review in self._poll_reviews(self.repo['organization'], self.repo['module'], branch, - start_id=start_id, is_open=True): + start_id=start_id, is_open=True, + grab_comments=grab_comments): yield review self.client.close() diff --git a/stackalytics/processor/record_processor.py b/stackalytics/processor/record_processor.py index 3562991ce..be180864d 100644 --- a/stackalytics/processor/record_processor.py +++ b/stackalytics/processor/record_processor.py @@ -473,25 +473,22 @@ class RecordProcessor(object): yield record - def _apply_type_based_processing(self, record): - if record['record_type'] == 'commit': - for r in self._process_commit(record): - yield r - elif record['record_type'] == 'review': - for r in self._process_review(record): - yield r - elif record['record_type'] == 'email': - for r in self._process_email(record): - yield r - elif record['record_type'] == 'bp': - for r in self._process_blueprint(record): - yield r - elif record['record_type'] == 'member': - for r in self._process_member(record): - yield r - elif record['record_type'] == 'bug': - for r in self._process_bug(record): - yield r + def _process_ci(self, record): + ci_vote = dict((k, v) for k, v in six.iteritems(record) + if k not in ['reviewer']) + + reviewer = record['reviewer'] + ci_vote['primary_key'] = ('%s:%s' % (reviewer['username'], + ci_vote['date'])) + ci_vote['user_id'] = reviewer['username'] + ci_vote['launchpad_id'] = reviewer['username'] + ci_vote['author_name'] = reviewer.get('name') or reviewer['username'] + ci_vote['author_email'] = ( + reviewer.get('email') or reviewer['username']).lower() + + self._update_record_and_user(ci_vote) + + yield ci_vote def _renew_record_date(self, record): record['week'] = utils.timestamp_to_week(record['date']) @@ -499,14 +496,19 @@ class RecordProcessor(object): record['release'] = self._get_release(record['date']) def process(self, record_iterator): + PROCESSORS = { + 'commit': self._process_commit, + 'review': self._process_review, + 'email': self._process_email, + 'bp': self._process_blueprint, + 'bug': self._process_bug, + 'member': self._process_member, + 'ci_vote': self._process_ci, + } + for record in record_iterator: - for r in self._apply_type_based_processing(record): - - if r['company_name'] == '*robots': - continue - + for r in PROCESSORS[record['record_type']](record): self._renew_record_date(r) - yield r def _update_records_with_releases(self, release_index):