diff --git a/dashboard/memory_storage.py b/dashboard/memory_storage.py index c6e42437e..a201a2cd7 100644 --- a/dashboard/memory_storage.py +++ b/dashboard/memory_storage.py @@ -33,6 +33,7 @@ class CachedMemoryStorage(MemoryStorage): self.user_id_index = {} self.company_index = {} self.release_index = {} + self.blueprint_id_index = {} self.indexes = { 'primary_key': self.primary_key_index, @@ -49,6 +50,11 @@ class CachedMemoryStorage(MemoryStorage): self.records[record['record_id']] = record for key, index in self.indexes.iteritems(): self._add_to_index(index, record, key) + for bp_id in (record.get('blueprint_id') or []): + if bp_id in self.blueprint_id_index: + self.blueprint_id_index[bp_id].add(record['record_id']) + else: + self.blueprint_id_index[bp_id] = set([record['record_id']]) def update(self, records): have_updates = False @@ -100,6 +106,10 @@ class CachedMemoryStorage(MemoryStorage): def get_record_ids_by_releases(self, releases): return self._get_record_ids_from_index(releases, self.release_index) + def get_record_ids_by_blueprint_ids(self, blueprint_ids): + return self._get_record_ids_from_index(blueprint_ids, + self.blueprint_id_index) + def get_record_ids(self): return self.records.keys() diff --git a/dashboard/templates/blueprint_report.html b/dashboard/templates/blueprint_report.html new file mode 100644 index 000000000..2b274e1c3 --- /dev/null +++ b/dashboard/templates/blueprint_report.html @@ -0,0 +1,75 @@ + + + + {{ blueprint.title }} + + + + + +

Blueprint “{{ blueprint.name }}”

+ +
Title: {{ blueprint.title }}
+
URL: https://blueprints.launchpad.net/{{ blueprint.module }}/+spec/{{ blueprint.name }}
+
Status: {{blueprint.lifecycle_status}}
+
Priority: {{blueprint.priority}}
+
Definition Status: {{blueprint.definition_status}}
+
Implementation Status: {{blueprint.implementation_status}}
+
Direction: {% if blueprint.direction_approved %} Approved {% else %} Needs Approval {% endif %}
+
Registered By: {{blueprint.author_name}} ({{ blueprint.company_name }})
+
Registered On: {{blueprint.date_str}}
+ +{% if blueprint.whiteboard %} +

Whiteboard

+
{{ blueprint.whiteboard }}
+{% endif %} + +

Activity Log

+ +{% if not activity %} +
No activities related to this blueprint.
+{% endif %} +{% for item in activity %} +
+
+
+
+
{{ item.date_str}}
+
{{ item.author_name }} ({{ item.company_name }})
+ {% if item.record_type == "commit" %} +
Commit “{{ item.subject }}”
+
{{ item.message }}
+
+{{ item.lines_added }} + - {{ item.lines_deleted }} +
+ {% if item.correction_comment %} +
Commit corrected: + {{ item.correction_comment }}
+ {% endif %} + {% elif item.record_type == "mark" %} +
Review “{{item.subject}}”
+
Patch submitted by {{ parent_author_link }}
+
Change Id: {{item.review_id}}
+
+ {{item.description}}: {{item.value}}
+ {% elif item.record_type == "email" %} +
Email “{{item.subject}}”
+ {% if item.body %} +
{{ item.body }}
+ {% endif %} + {% endif %} +
+
+ +{% endfor %} + + \ No newline at end of file diff --git a/dashboard/templates/overview.html b/dashboard/templates/overview.html index e0745dafc..49eafc32e 100644 --- a/dashboard/templates/overview.html +++ b/dashboard/templates/overview.html @@ -43,6 +43,11 @@ $('#activity_header').hide(); } $("#activity_template").tmpl(data["activity"]).appendTo("#activity_container"); + $('.ext_link').click(function(event) { + event.preventDefault(); + event.stopPropagation(); + window.open(this.href, '_blank'); + }); } }); } @@ -90,7 +95,7 @@
{%html author_link %} ({%html company_link %})
-
${date_str} to ${module}
+
${date_str} in {%html module_link%}
{%if record_type == "commit" %} @@ -125,7 +130,7 @@
{%/if%} {%elif ((record_type == "bpd") || (record_type == "bpc")) %} -
${title} (${name})
+
${title} (${name})
${summary}
Priority: ${priority}
diff --git a/dashboard/web.py b/dashboard/web.py index 98694be5f..bb5b4b01c 100644 --- a/dashboard/web.py +++ b/dashboard/web.py @@ -49,7 +49,7 @@ METRIC_LABELS = { 'commits': 'Commits', 'marks': 'Reviews', 'emails': 'Emails', - 'bpd': 'New Blueprints', + 'bpd': 'Drafted Blueprints', 'bpc': 'Completed Blueprints', } @@ -615,6 +615,9 @@ def _extend_record(record): record['company_link'] = make_link( record['company_name'], '/', {'company': record['company_name'], 'user_id': ''}) + record['module_link'] = make_link( + record['module'], '/', + {'module': record['module'], 'company': '', 'user_id': ''}) record['gravatar'] = gravatar(record.get('author_email', 'stackalytics')) record['blueprint_id_count'] = len(record.get('blueprint_id', [])) record['bug_id_count'] = len(record.get('bug_id', [])) @@ -662,7 +665,7 @@ def get_activity_json(records): blueprint = record.copy() _extend_record(blueprint) if 'mention_date' in record: - record['mention_date_str'] = format_datetime( + blueprint['mention_date_str'] = format_datetime( record['mention_date']) result.append(blueprint) @@ -904,12 +907,46 @@ def get_commit_report(records): return response +@app.route('/report/blueprint//') +@templated() +@exception_handler() +def blueprint_report(module, blueprint_name): + memory_storage_inst = get_vault()['memory_storage'] + runtime_storage_inst = get_vault()['runtime_storage'] + + blueprint_id = module + ':' + blueprint_name + + for bpd in memory_storage_inst.get_records( + memory_storage_inst.get_record_ids_by_type('bpd')): + if bpd['id'] == blueprint_id: + _extend_record(bpd) + break + else: + flask.abort(404) + return + + record_ids = memory_storage_inst.get_record_ids_by_blueprint_ids( + [blueprint_id]) + + activity = [] + for record in memory_storage_inst.get_records(record_ids): + _extend_record(record) + if record['record_type'] == 'email': + record['body'] = (runtime_storage_inst.get_by_key('email:%s' % + record['primary_key'])) + activity.append(record) + + activity.sort(key=lambda x: x['date']) + + return {'blueprint': bpd, 'activity': activity} + + # Jinja Filters --------- @app.template_filter('datetimeformat') def format_datetime(timestamp): return datetime.datetime.utcfromtimestamp( - timestamp).strftime('%d %b %Y @ %H:%M') + timestamp).strftime('%d %b %Y %H:%M:%S') @app.template_filter('launchpadmodule') @@ -960,15 +997,18 @@ def make_commit_message(record): # clear text s = cgi.escape(re.sub(re.compile('\n{2,}', flags=re.MULTILINE), '\n', s)) + s = re.sub(r'([/\/]+)', r'\1​', s) # insert links s = re.sub(re.compile('(blueprint\s+)([\w-]+)', flags=re.IGNORECASE), r'\1\2', s) - s = re.sub(re.compile('(bug\s+)#?([\d]{5,7})', flags=re.IGNORECASE), - r'\1\2', s) + module + r'/+spec/\2" class="ext_link">\2', s) + s = re.sub(re.compile('(bug[\s#:]*)([\d]{5,7})', flags=re.IGNORECASE), + r'\1\2', s) s = re.sub(r'\s+(I[0-9a-f]{40})', - r' \1', s) + r' \1', s) s = unwrap_text(s) return s diff --git a/stackalytics/processor/lp.py b/stackalytics/processor/lp.py index 94a9c919c..a16f27b70 100644 --- a/stackalytics/processor/lp.py +++ b/stackalytics/processor/lp.py @@ -49,6 +49,7 @@ def log(repo): record[field] = utils.iso8601_to_timestamp(date) record['module'] = module + record['id'] = module + ':' + record['name'] LOG.debug('New blueprint: %s', record) yield record diff --git a/stackalytics/processor/main.py b/stackalytics/processor/main.py index 00323ab1f..3c676b150 100644 --- a/stackalytics/processor/main.py +++ b/stackalytics/processor/main.py @@ -17,7 +17,6 @@ import urllib from oslo.config import cfg import psutil -from psutil import _error from stackalytics.openstack.common import log as logging from stackalytics.processor import config @@ -42,8 +41,8 @@ def get_pids(): if p.cmdline and p.cmdline[0].find('/uwsgi'): if p.parent: uwsgi_dict[p.pid] = p.parent.pid - except _error.NoSuchProcess: - # the process may disappear after get_pid_list call, ignore it + except Exception as e: + LOG.debug('Exception while iterating process list: %s', e) pass result = set() diff --git a/stackalytics/processor/mls.py b/stackalytics/processor/mls.py index d38324b89..47c5bfbe7 100644 --- a/stackalytics/processor/mls.py +++ b/stackalytics/processor/mls.py @@ -19,7 +19,6 @@ import StringIO from email import utils as email_utils import re -import time import urlparse from stackalytics.openstack.common import log as logging @@ -81,6 +80,9 @@ def _link_content_changed(link, runtime_storage_inst): def _retrieve_mails(uri): LOG.debug('Retrieving mail archive from uri: %s', uri) content = utils.read_uri(uri) + if not content: + LOG.error('Error reading mail archive from uri: %s', uri) + return gzip_fd = gzip.GzipFile(fileobj=StringIO.StringIO(content)) content = gzip_fd.read() LOG.debug('Mail archive is loaded, start processing') @@ -94,7 +96,8 @@ def _retrieve_mails(uri): continue author_name = rec.group(2) - date = int(time.mktime(email_utils.parsedate(rec.group(3)))) + date = int(email_utils.mktime_tz( + email_utils.parsedate_tz(rec.group(3)))) subject = rec.group(4) message_id = rec.group(5) body = rec.group(6) @@ -105,15 +108,18 @@ def _retrieve_mails(uri): 'author_email': author_email, 'subject': subject, 'date': date, + 'body': body, } for pattern_name, pattern in MESSAGE_PATTERNS.iteritems(): collection = set() for item in re.finditer(pattern, body): groups = item.groupdict() - collection.add(groups['id']) + item_id = groups['id'] if 'module' in groups: + item_id = groups['module'] + ':' + item_id email['module'] = groups['module'] + collection.add(item_id) email[pattern_name] = list(collection) yield email @@ -125,5 +131,5 @@ def log(uri, runtime_storage_inst): for link in links: if _link_content_changed(link, runtime_storage_inst): for mail in _retrieve_mails(link): - LOG.debug('New mail: %s', mail) + LOG.debug('New mail: %s', mail['message_id']) yield mail diff --git a/stackalytics/processor/record_processor.py b/stackalytics/processor/record_processor.py index 284084479..7644ad233 100644 --- a/stackalytics/processor/record_processor.py +++ b/stackalytics/processor/record_processor.py @@ -282,14 +282,21 @@ class RecordProcessor(object): self._update_record_and_user(record) self._guess_module(record) + if record.get('blueprint_id'): + self.runtime_storage_inst.set_by_key( + 'email:%s' % record['primary_key'], record['body']) + + del record['body'] + yield record def _process_blueprint(self, record): bpd_author = record.get('drafter') or record.get('owner') - bpd = dict([(k, v) for k, v in record.iteritems()]) + bpd = dict([(k, v) for k, v in record.iteritems() + if k.find('_link') < 0]) bpd['record_type'] = 'bpd' - bpd['primary_key'] = 'bpd:' + record['self_link'] + bpd['primary_key'] = 'bpd:' + record['id'] bpd['launchpad_id'] = bpd_author bpd['date'] = record['date_created'] @@ -298,9 +305,10 @@ class RecordProcessor(object): yield bpd if record.get('assignee') and record['date_completed']: - bpc = dict([(k, v) for k, v in record.iteritems()]) + bpc = dict([(k, v) for k, v in record.iteritems() + if k.find('_link') < 0]) bpc['record_type'] = 'bpc' - bpc['primary_key'] = 'bpc:' + record['self_link'] + bpc['primary_key'] = 'bpc:' + record['id'] bpc['launchpad_id'] = record['assignee'] bpc['date'] = record['date_completed'] @@ -381,7 +389,7 @@ class RecordProcessor(object): 'date': record['date'] } if record['record_type'] in ['bpd', 'bpi']: - valid_blueprints[record['name']] = { + valid_blueprints[record['id']] = { 'primary_key': record['primary_key'], 'count': 0, 'date': record['date'] @@ -419,7 +427,7 @@ class RecordProcessor(object): record['blueprint_id'] = list(valid_bp) if record['record_type'] in ['bpd', 'bpi']: - bp = valid_blueprints[record['name']] + bp = valid_blueprints[record['id']] if ((record.get('mention_count') != bp['count']) or (record.get('mention_date') != bp['date'])): record['mention_count'] = bp['count'] diff --git a/stackalytics/processor/vcs.py b/stackalytics/processor/vcs.py index d568571a2..c0ec7df5b 100644 --- a/stackalytics/processor/vcs.py +++ b/stackalytics/processor/vcs.py @@ -175,6 +175,10 @@ class Git(Vcs): commit['release'] = self.release_index[commit['commit_id']] else: commit['release'] = None + if 'blueprint_id' in commit: + commit['blueprint_id'] = [(commit['module'] + ':' + bp_name) + for bp_name + in commit['blueprint_id']] yield commit diff --git a/tests/unit/test_record_processor.py b/tests/unit/test_record_processor.py index 254c40380..8296884b7 100644 --- a/tests/unit/test_record_processor.py +++ b/tests/unit/test_record_processor.py @@ -325,6 +325,7 @@ class TestRecordProcessor(testtools.TestCase): processed_records = list(record_processor_inst.process([ {'record_type': 'bp', + 'id': 'mod:blueprint', 'self_link': 'http://launchpad.net/blueprint', 'owner': 'john_doe', 'date_created': 1234567890} @@ -358,6 +359,7 @@ class TestRecordProcessor(testtools.TestCase): processed_records = list(record_processor_inst.process([ {'record_type': 'bp', + 'id': 'mod:blueprint', 'self_link': 'http://launchpad.net/blueprint', 'owner': 'john_doe', 'date_created': 1234567890} @@ -387,6 +389,7 @@ class TestRecordProcessor(testtools.TestCase): processed_records = list(record_processor_inst.process([ {'record_type': 'bp', + 'id': 'mod:blueprint', 'self_link': 'http://launchpad.net/blueprint', 'owner': 'john_doe', 'date_created': 1234567890}, @@ -432,6 +435,7 @@ class TestRecordProcessor(testtools.TestCase): processed_records = list(record_processor_inst.process([ {'record_type': 'bp', + 'id': 'mod:blueprint', 'self_link': 'http://launchpad.net/blueprint', 'owner': 'john_doe', 'date_created': 1234567890}, @@ -483,6 +487,7 @@ class TestRecordProcessor(testtools.TestCase): 'createdOn': 1379404951, 'module': 'nova'}, {'record_type': 'bp', + 'id': 'mod:blueprint', 'self_link': 'http://launchpad.net/blueprint', 'owner': 'john_doe', 'date_created': 1234567890} @@ -777,6 +782,7 @@ def generate_emails(author_name='John Doe', author_email='johndoe@gmail.com', 'date': date, 'subject': subject, 'module': module, + 'body': 'lorem ipsum', } diff --git a/tests/unit/test_vcs.py b/tests/unit/test_vcs.py index 4954e9567..fc99bad78 100644 --- a/tests/unit/test_vcs.py +++ b/tests/unit/test_vcs.py @@ -122,7 +122,7 @@ diff_stat: self.assertEquals(0, commits[3]['files_changed']) self.assertEquals(0, commits[3]['lines_added']) self.assertEquals(0, commits[3]['lines_deleted']) - self.assertEquals(set(['fix-me']), + self.assertEquals(set(['dummy:fix-me']), set(commits[3]['blueprint_id'])) self.assertEquals(0, commits[4]['files_changed']) diff --git a/tests/unit/test_web_utils.py b/tests/unit/test_web_utils.py index f9b005b28..7a657d1da 100644 --- a/tests/unit/test_web_utils.py +++ b/tests/unit/test_web_utils.py @@ -44,9 +44,10 @@ Change-Id: Ie49ccd2138905e178843b375a9b16c3fe572d1db''' During finish_migration the manager calls initialize_connection but doesn't \ update the block_device_mapping with the potentially new connection_info \ returned. -Fixes bug 1076801 +Fixes bug \ +1076801 ''' + ('Change-Id: ' + 'Ie49ccd2138905e178843b375a9b16c3fe572d1db,n,z" class="ext_link">' 'Ie49ccd2138905e178843b375a9b16c3fe572d1db') observed = web.make_commit_message(record) @@ -71,9 +72,9 @@ Change-Id: Ie49ccd2138905e178843b375a9b16c3fe572d1db''' Implemented new driver for Cinder <: Implements Blueprint ''' + ( 'super-driver' + '\n' + + 'super-driver" class="ext_link">super-driver' + '\n' + 'Change-Id: ' + 'Ie49ccd2138905e178843b375a9b16c3fe572d1db,n,z" class="ext_link">' 'Ie49ccd2138905e178843b375a9b16c3fe572d1db') observed = web.make_commit_message(record) @@ -87,6 +88,15 @@ Implements Blueprint ''' + ( self.assertEqual(expected, web.unwrap_text(original)) + def test_unwrap_split_long_link(self): + original = ('https://blueprints.launchpad.net/stackalytics/+spec/' + 'stackalytics-core') + expected = ('https://​blueprints.launchpad.net/​' + 'stackalytics/​+spec/​stackalytics-core') + + self.assertEqual(expected, web.make_commit_message( + {'message': original, 'module': 'none'})) + @mock.patch('dashboard.web.get_vault') @mock.patch('dashboard.web.get_user_from_runtime_storage') def test_make_page_title(self, user_patch, vault_patch):