Implemented report of blueprint activity

* Introduced report that shows blueprint details followed by list
of commits, emails and reviews related to that blueprint
* Mail body is stored in runtime storage
* External links are opened in new window/tab

Implements blueprint report-blueprint-activity

Change-Id: I1515d7fc51a4b46932fc15551c94f0fa6b16e4ed
This commit is contained in:
Ilya Shakhat 2013-09-23 11:18:28 +04:00
parent 6b19d37082
commit 240e9406b4
12 changed files with 191 additions and 27 deletions

View File

@ -33,6 +33,7 @@ class CachedMemoryStorage(MemoryStorage):
self.user_id_index = {}
self.company_index = {}
self.release_index = {}
self.blueprint_id_index = {}
self.indexes = {
'primary_key': self.primary_key_index,
@ -49,6 +50,11 @@ class CachedMemoryStorage(MemoryStorage):
self.records[record['record_id']] = record
for key, index in self.indexes.iteritems():
self._add_to_index(index, record, key)
for bp_id in (record.get('blueprint_id') or []):
if bp_id in self.blueprint_id_index:
self.blueprint_id_index[bp_id].add(record['record_id'])
else:
self.blueprint_id_index[bp_id] = set([record['record_id']])
def update(self, records):
have_updates = False
@ -100,6 +106,10 @@ class CachedMemoryStorage(MemoryStorage):
def get_record_ids_by_releases(self, releases):
return self._get_record_ids_from_index(releases, self.release_index)
def get_record_ids_by_blueprint_ids(self, blueprint_ids):
return self._get_record_ids_from_index(blueprint_ids,
self.blueprint_id_index)
def get_record_ids(self):
return self.records.keys()

View File

@ -0,0 +1,75 @@
<!DOCTYPE html>
<html>
<head>
<title>{{ blueprint.title }}</title>
<link rel=stylesheet type=text/css href="{{ url_for('static', filename='css/style.css') }}">
<style>
.label {
font-weight: bold;
line-height: 135%;
}
.message {
margin-top: 1em;
white-space: pre-wrap;
}
</style>
</head>
<body style="margin: 2em;">
<h1>Blueprint &ldquo;{{ blueprint.name }}&rdquo;</h1>
<div><span class="label">Title:</span> {{ blueprint.title }}</div>
<div><span class="label">URL:</span> <a href="https://blueprints.launchpad.net/{{ blueprint.module }}/+spec/{{ blueprint.name }}">https://blueprints.launchpad.net/{{ blueprint.module }}/+spec/{{ blueprint.name }}</a></div>
<div><span class="label">Status:</span> <span class="status{{blueprint.lifecycle_status}}">{{blueprint.lifecycle_status}}</span></div>
<div><span class="label">Priority:</span> <span class="status{{blueprint.priority}}">{{blueprint.priority}}</span></div>
<div><span class="label">Definition Status:</span> <span class="specstatus{{blueprint.definition_status}}">{{blueprint.definition_status}}</span></div>
<div><span class="label">Implementation Status:</span> <span class="specdelivery{{blueprint.implementation_status}}">{{blueprint.implementation_status}}</span></div>
<div><span class="label">Direction:</span> {% if blueprint.direction_approved %} Approved {% else %} Needs Approval {% endif %}</div>
<div><span class="label">Registered By:</span> {{blueprint.author_name}} ({{ blueprint.company_name }})</div>
<div><span class="label">Registered On:</span> {{blueprint.date_str}}</div>
{% if blueprint.whiteboard %}
<h2>Whiteboard</h2>
<div class="message">{{ blueprint.whiteboard }}</div>
{% endif %}
<h2>Activity Log</h2>
{% if not activity %}
<div>No activities related to this blueprint.</div>
{% endif %}
{% for item in activity %}
<div style="margin-bottom: 1em;">
<div style='float: left; '><img src="{{ item.author_email | gravatar(size=64) }}">
</div>
<div style="margin-left: 80px;">
<div style="font-weight: bold;">{{ item.date_str}}</div>
<div style="font-weight: bold;">{{ item.author_name }} ({{ item.company_name }})</div>
{% if item.record_type == "commit" %}
<div style='font-weight: bold;'>Commit &ldquo;{{ item.subject }}&rdquo;</div>
<div class="message">{{ item.message }}</div>
<div><span style="color: green">+<span>{{ item.lines_added }}</span></span>
<span style="color: blue">- <span>{{ item.lines_deleted }}</span></span>
</div>
{% if item.correction_comment %}
<div style='font-weight: bold; color: red;'>Commit corrected:
<span>{{ item.correction_comment }}</span></div>
{% endif %}
{% elif item.record_type == "mark" %}
<div style='font-weight: bold;'>Review &ldquo;{{item.subject}}&rdquo;</div>
<div>Patch submitted by {{ parent_author_link }}</div>
<div>Change Id: <a href="{{item.url}}">{{item.review_id}}</a></div>
<div style="color: {% if item.value > 0 %} green {% else %} blue {% endif %}">
{{item.description}}: <span class="review_mark">{{item.value}}</span></div>
{% elif item.record_type == "email" %}
<div style='font-weight: bold;'>Email &ldquo;{{item.subject}}&rdquo;</div>
{% if item.body %}
<div class="message">{{ item.body }}</div>
{% endif %}
{% endif %}
</div>
</div>
{% endfor %}
</body>
</html>

View File

@ -43,6 +43,11 @@
$('#activity_header').hide();
}
$("#activity_template").tmpl(data["activity"]).appendTo("#activity_container");
$('.ext_link').click(function(event) {
event.preventDefault();
event.stopPropagation();
window.open(this.href, '_blank');
});
}
});
}
@ -90,7 +95,7 @@
<div style='float: left; '><img src="${gravatar}" style="width: 32px; height: 32px;"></div>
<div style="margin-left: 40px;">
<div style="font-weight: bold;">{%html author_link %} ({%html company_link %})</div>
<div style="font-weight: bold;">${date_str} to <a href="https://launchpad.net/${module}">${module}</a></div>
<div style="font-weight: bold;">${date_str} in {%html module_link%}</div>
</div>
<div style="margin-left: 40px;">
{%if record_type == "commit" %}
@ -125,7 +130,7 @@
</div>
{%/if%}
{%elif ((record_type == "bpd") || (record_type == "bpc")) %}
<div style='font-weight: bold;'>${title} (<a href='${web_link}'>${name}</a>)</div>
<div style='font-weight: bold;'>${title} (<a href="/report/blueprint/${module}/${name}" class="ext_link">${name}</a>)</div>
<div style='white-space: pre-wrap;'>${summary}</div>
<div>Priority: <span class="specpriority${priority}">${priority}</span></div>

View File

@ -49,7 +49,7 @@ METRIC_LABELS = {
'commits': 'Commits',
'marks': 'Reviews',
'emails': 'Emails',
'bpd': 'New Blueprints',
'bpd': 'Drafted Blueprints',
'bpc': 'Completed Blueprints',
}
@ -615,6 +615,9 @@ def _extend_record(record):
record['company_link'] = make_link(
record['company_name'], '/',
{'company': record['company_name'], 'user_id': ''})
record['module_link'] = make_link(
record['module'], '/',
{'module': record['module'], 'company': '', 'user_id': ''})
record['gravatar'] = gravatar(record.get('author_email', 'stackalytics'))
record['blueprint_id_count'] = len(record.get('blueprint_id', []))
record['bug_id_count'] = len(record.get('bug_id', []))
@ -662,7 +665,7 @@ def get_activity_json(records):
blueprint = record.copy()
_extend_record(blueprint)
if 'mention_date' in record:
record['mention_date_str'] = format_datetime(
blueprint['mention_date_str'] = format_datetime(
record['mention_date'])
result.append(blueprint)
@ -904,12 +907,46 @@ def get_commit_report(records):
return response
@app.route('/report/blueprint/<module>/<blueprint_name>')
@templated()
@exception_handler()
def blueprint_report(module, blueprint_name):
memory_storage_inst = get_vault()['memory_storage']
runtime_storage_inst = get_vault()['runtime_storage']
blueprint_id = module + ':' + blueprint_name
for bpd in memory_storage_inst.get_records(
memory_storage_inst.get_record_ids_by_type('bpd')):
if bpd['id'] == blueprint_id:
_extend_record(bpd)
break
else:
flask.abort(404)
return
record_ids = memory_storage_inst.get_record_ids_by_blueprint_ids(
[blueprint_id])
activity = []
for record in memory_storage_inst.get_records(record_ids):
_extend_record(record)
if record['record_type'] == 'email':
record['body'] = (runtime_storage_inst.get_by_key('email:%s' %
record['primary_key']))
activity.append(record)
activity.sort(key=lambda x: x['date'])
return {'blueprint': bpd, 'activity': activity}
# Jinja Filters ---------
@app.template_filter('datetimeformat')
def format_datetime(timestamp):
return datetime.datetime.utcfromtimestamp(
timestamp).strftime('%d %b %Y @ %H:%M')
timestamp).strftime('%d %b %Y %H:%M:%S')
@app.template_filter('launchpadmodule')
@ -960,15 +997,18 @@ def make_commit_message(record):
# clear text
s = cgi.escape(re.sub(re.compile('\n{2,}', flags=re.MULTILINE), '\n', s))
s = re.sub(r'([/\/]+)', r'\1&#8203;', s)
# insert links
s = re.sub(re.compile('(blueprint\s+)([\w-]+)', flags=re.IGNORECASE),
r'\1<a href="https://blueprints.launchpad.net/' +
module + r'/+spec/\2">\2</a>', s)
s = re.sub(re.compile('(bug\s+)#?([\d]{5,7})', flags=re.IGNORECASE),
r'\1<a href="https://bugs.launchpad.net/bugs/\2">\2</a>', s)
module + r'/+spec/\2" class="ext_link">\2</a>', s)
s = re.sub(re.compile('(bug[\s#:]*)([\d]{5,7})', flags=re.IGNORECASE),
r'\1<a href="https://bugs.launchpad.net/bugs/\2" '
r'class="ext_link">\2</a>', s)
s = re.sub(r'\s+(I[0-9a-f]{40})',
r' <a href="https://review.openstack.org/#q,\1,n,z">\1</a>', s)
r' <a href="https://review.openstack.org/#q,\1,n,z" '
r'class="ext_link">\1</a>', s)
s = unwrap_text(s)
return s

View File

@ -49,6 +49,7 @@ def log(repo):
record[field] = utils.iso8601_to_timestamp(date)
record['module'] = module
record['id'] = module + ':' + record['name']
LOG.debug('New blueprint: %s', record)
yield record

View File

@ -17,7 +17,6 @@ import urllib
from oslo.config import cfg
import psutil
from psutil import _error
from stackalytics.openstack.common import log as logging
from stackalytics.processor import config
@ -42,8 +41,8 @@ def get_pids():
if p.cmdline and p.cmdline[0].find('/uwsgi'):
if p.parent:
uwsgi_dict[p.pid] = p.parent.pid
except _error.NoSuchProcess:
# the process may disappear after get_pid_list call, ignore it
except Exception as e:
LOG.debug('Exception while iterating process list: %s', e)
pass
result = set()

View File

@ -19,7 +19,6 @@ import StringIO
from email import utils as email_utils
import re
import time
import urlparse
from stackalytics.openstack.common import log as logging
@ -81,6 +80,9 @@ def _link_content_changed(link, runtime_storage_inst):
def _retrieve_mails(uri):
LOG.debug('Retrieving mail archive from uri: %s', uri)
content = utils.read_uri(uri)
if not content:
LOG.error('Error reading mail archive from uri: %s', uri)
return
gzip_fd = gzip.GzipFile(fileobj=StringIO.StringIO(content))
content = gzip_fd.read()
LOG.debug('Mail archive is loaded, start processing')
@ -94,7 +96,8 @@ def _retrieve_mails(uri):
continue
author_name = rec.group(2)
date = int(time.mktime(email_utils.parsedate(rec.group(3))))
date = int(email_utils.mktime_tz(
email_utils.parsedate_tz(rec.group(3))))
subject = rec.group(4)
message_id = rec.group(5)
body = rec.group(6)
@ -105,15 +108,18 @@ def _retrieve_mails(uri):
'author_email': author_email,
'subject': subject,
'date': date,
'body': body,
}
for pattern_name, pattern in MESSAGE_PATTERNS.iteritems():
collection = set()
for item in re.finditer(pattern, body):
groups = item.groupdict()
collection.add(groups['id'])
item_id = groups['id']
if 'module' in groups:
item_id = groups['module'] + ':' + item_id
email['module'] = groups['module']
collection.add(item_id)
email[pattern_name] = list(collection)
yield email
@ -125,5 +131,5 @@ def log(uri, runtime_storage_inst):
for link in links:
if _link_content_changed(link, runtime_storage_inst):
for mail in _retrieve_mails(link):
LOG.debug('New mail: %s', mail)
LOG.debug('New mail: %s', mail['message_id'])
yield mail

View File

@ -282,14 +282,21 @@ class RecordProcessor(object):
self._update_record_and_user(record)
self._guess_module(record)
if record.get('blueprint_id'):
self.runtime_storage_inst.set_by_key(
'email:%s' % record['primary_key'], record['body'])
del record['body']
yield record
def _process_blueprint(self, record):
bpd_author = record.get('drafter') or record.get('owner')
bpd = dict([(k, v) for k, v in record.iteritems()])
bpd = dict([(k, v) for k, v in record.iteritems()
if k.find('_link') < 0])
bpd['record_type'] = 'bpd'
bpd['primary_key'] = 'bpd:' + record['self_link']
bpd['primary_key'] = 'bpd:' + record['id']
bpd['launchpad_id'] = bpd_author
bpd['date'] = record['date_created']
@ -298,9 +305,10 @@ class RecordProcessor(object):
yield bpd
if record.get('assignee') and record['date_completed']:
bpc = dict([(k, v) for k, v in record.iteritems()])
bpc = dict([(k, v) for k, v in record.iteritems()
if k.find('_link') < 0])
bpc['record_type'] = 'bpc'
bpc['primary_key'] = 'bpc:' + record['self_link']
bpc['primary_key'] = 'bpc:' + record['id']
bpc['launchpad_id'] = record['assignee']
bpc['date'] = record['date_completed']
@ -381,7 +389,7 @@ class RecordProcessor(object):
'date': record['date']
}
if record['record_type'] in ['bpd', 'bpi']:
valid_blueprints[record['name']] = {
valid_blueprints[record['id']] = {
'primary_key': record['primary_key'],
'count': 0,
'date': record['date']
@ -419,7 +427,7 @@ class RecordProcessor(object):
record['blueprint_id'] = list(valid_bp)
if record['record_type'] in ['bpd', 'bpi']:
bp = valid_blueprints[record['name']]
bp = valid_blueprints[record['id']]
if ((record.get('mention_count') != bp['count']) or
(record.get('mention_date') != bp['date'])):
record['mention_count'] = bp['count']

View File

@ -175,6 +175,10 @@ class Git(Vcs):
commit['release'] = self.release_index[commit['commit_id']]
else:
commit['release'] = None
if 'blueprint_id' in commit:
commit['blueprint_id'] = [(commit['module'] + ':' + bp_name)
for bp_name
in commit['blueprint_id']]
yield commit

View File

@ -325,6 +325,7 @@ class TestRecordProcessor(testtools.TestCase):
processed_records = list(record_processor_inst.process([
{'record_type': 'bp',
'id': 'mod:blueprint',
'self_link': 'http://launchpad.net/blueprint',
'owner': 'john_doe',
'date_created': 1234567890}
@ -358,6 +359,7 @@ class TestRecordProcessor(testtools.TestCase):
processed_records = list(record_processor_inst.process([
{'record_type': 'bp',
'id': 'mod:blueprint',
'self_link': 'http://launchpad.net/blueprint',
'owner': 'john_doe',
'date_created': 1234567890}
@ -387,6 +389,7 @@ class TestRecordProcessor(testtools.TestCase):
processed_records = list(record_processor_inst.process([
{'record_type': 'bp',
'id': 'mod:blueprint',
'self_link': 'http://launchpad.net/blueprint',
'owner': 'john_doe',
'date_created': 1234567890},
@ -432,6 +435,7 @@ class TestRecordProcessor(testtools.TestCase):
processed_records = list(record_processor_inst.process([
{'record_type': 'bp',
'id': 'mod:blueprint',
'self_link': 'http://launchpad.net/blueprint',
'owner': 'john_doe',
'date_created': 1234567890},
@ -483,6 +487,7 @@ class TestRecordProcessor(testtools.TestCase):
'createdOn': 1379404951,
'module': 'nova'},
{'record_type': 'bp',
'id': 'mod:blueprint',
'self_link': 'http://launchpad.net/blueprint',
'owner': 'john_doe',
'date_created': 1234567890}
@ -777,6 +782,7 @@ def generate_emails(author_name='John Doe', author_email='johndoe@gmail.com',
'date': date,
'subject': subject,
'module': module,
'body': 'lorem ipsum',
}

View File

@ -122,7 +122,7 @@ diff_stat:
self.assertEquals(0, commits[3]['files_changed'])
self.assertEquals(0, commits[3]['lines_added'])
self.assertEquals(0, commits[3]['lines_deleted'])
self.assertEquals(set(['fix-me']),
self.assertEquals(set(['dummy:fix-me']),
set(commits[3]['blueprint_id']))
self.assertEquals(0, commits[4]['files_changed'])

View File

@ -44,9 +44,10 @@ Change-Id: Ie49ccd2138905e178843b375a9b16c3fe572d1db'''
During finish_migration the manager calls initialize_connection but doesn't \
update the block_device_mapping with the potentially new connection_info \
returned.
Fixes bug <a href="https://bugs.launchpad.net/bugs/1076801">1076801</a>
Fixes bug <a href="https://bugs.launchpad.net/bugs/1076801" class="ext_link">\
1076801</a>
''' + ('Change-Id: <a href="https://review.openstack.org/#q,'
'Ie49ccd2138905e178843b375a9b16c3fe572d1db,n,z">'
'Ie49ccd2138905e178843b375a9b16c3fe572d1db,n,z" class="ext_link">'
'Ie49ccd2138905e178843b375a9b16c3fe572d1db</a>')
observed = web.make_commit_message(record)
@ -71,9 +72,9 @@ Change-Id: Ie49ccd2138905e178843b375a9b16c3fe572d1db'''
Implemented new driver for Cinder &lt;:
Implements Blueprint ''' + (
'<a href="https://blueprints.launchpad.net/cinder/+spec/'
'super-driver">super-driver</a>' + '\n' +
'super-driver" class="ext_link">super-driver</a>' + '\n' +
'Change-Id: <a href="https://review.openstack.org/#q,'
'Ie49ccd2138905e178843b375a9b16c3fe572d1db,n,z">'
'Ie49ccd2138905e178843b375a9b16c3fe572d1db,n,z" class="ext_link">'
'Ie49ccd2138905e178843b375a9b16c3fe572d1db</a>')
observed = web.make_commit_message(record)
@ -87,6 +88,15 @@ Implements Blueprint ''' + (
self.assertEqual(expected, web.unwrap_text(original))
def test_unwrap_split_long_link(self):
original = ('https://blueprints.launchpad.net/stackalytics/+spec/'
'stackalytics-core')
expected = ('https://&#8203;blueprints.launchpad.net/&#8203;'
'stackalytics/&#8203;+spec/&#8203;stackalytics-core')
self.assertEqual(expected, web.make_commit_message(
{'message': original, 'module': 'none'}))
@mock.patch('dashboard.web.get_vault')
@mock.patch('dashboard.web.get_user_from_runtime_storage')
def test_make_page_title(self, user_patch, vault_patch):