x/stackalytics
Code Issues Proposed changes

Implementation of blueprint stackalytics-core

Browse Source 
* Corrected work with default parameters
* Fixed module and engineer details screens
* Map all robots to *robots company

Change-Id: I989e09c04c12f3a0a9035a8ddcd730d93d0ce12f
This commit is contained in:
Ilya Shakhat 2013-07-11 13:39:17 +04:00
parent b7f19335f6
commit 6d89020cab
9 changed files with 304 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
dashboard/memory_storage.py
View File

@ -20,8 +20,9 @@ class CachedMemoryStorage(MemoryStorage):
self.release_index = {}
self.dates = []
for record in records:
self.records[record['record_id']] = record
self.index(record)
if record['company_name'] != '*robots': # ignore robots
self.records[record['record_id']] = record
self.index(record)
self.dates = sorted(self.date_index)
self.company_name_mapping = dict((c.lower(), c)
for c in self.company_index.keys())

2
dashboard/templates/base.html
View File

@ -133,7 +133,7 @@
</table>
<div id="analytics_header">
<h3><a href="/?metric={{ metric }}&period={{ period }}&project_type={{ project_type }}">Stackalytics</a> | {{ self.title() }}</h3>
<h3><a href="/?metric={{ metric }}&release={{ release }}&project_type={{ project_type }}">Stackalytics</a> | {{ self.title() }}</h3>
<p>Community heartbeat</p>
</div>
<!--#topdynamicinner-->

2
dashboard/templates/engineer_details.html
View File

@ -26,7 +26,7 @@
<h3>Commits history</h3>
{% if not commits %}
<div>There are no commits for selected period or project type.</div>
<div>There are no commits for selected release or project type.</div>
{% endif %}
{% for rec in commits %}

19
dashboard/templates/layout.html
View File

@ -117,7 +117,11 @@
} else {
index++;
}
var link = make_link(link_prefix, data[i].id, data[i].name);
if (data[i].id) {
var link = make_link(link_prefix, data[i].id, data[i].name);
} else {
var link = data[i].name
}
tableData.push({"index": index_label, "link": link, "metric": data[i].metric});
}
@ -197,12 +201,12 @@
{# if (getRelease() != 'havana') {#}
options['release'] = getRelease();
{# }#}
if (getMetric() != 'loc') {
{# if (getMetric() != 'loc') {#}
options['metric'] = getMetric();
}
if (getProjectType() != 'incubation') {
{# }#}
{# if (getProjectType() != 'incubation') {#}
options['project_type'] = getProjectType();
}
{# }#}
return options;
}
@ -249,9 +253,8 @@
<div class="drops" style='margin: 0.8em; height: 2em;'>
<span class="drop_metric" style="float: right;">
<label for="project_type">Projects&nbsp;</label><select id="project_type" name="project_type">
<option value="core">Core</option>
<option value="incubation">Core+Incubation</option>
<option value="all">All</option>
<option value="openstack">OpenStack</option>
<option value="stackforge">StackForge</option>
</select>
</span>
<span class="drop_metric" style="float: right;">

4
dashboard/templates/module_details.html
View File

@ -6,8 +6,8 @@
{% block scripts %}
<script type="text/javascript">
chartAndTableRenderer("/data/companies", "left_list", "left_chart", "/companies/", {company: "{{ company|encode }}" });
timelineRenderer({company: "{{ company|encode }}" })
chartAndTableRenderer("/data/companies", "left_list", "left_chart", "/companies/", {module: "{{ module }}" });
timelineRenderer({module: "{{ module }}" })
</script>
{% endblock %}

217
dashboard/web.py
View File

@ -1,3 +1,18 @@
# Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import cgi
import datetime
import functools
@ -41,6 +56,9 @@ def get_vault():
releases = vault['persistent_storage'].get_releases()
vault['releases'] = dict((r['release_name'].lower(), r)
for r in releases)
modules = vault['persistent_storage'].get_repos()
vault['modules'] = dict((r['module'].lower(),
r['project_type'].lower()) for r in modules)
app.stackalytics_vault = vault
return vault
@ -49,7 +67,32 @@ def get_memory_storage():
return get_vault()['memory_storage']
def record_filter(parameter_getter=lambda x: flask.request.args.get(x)):
def get_default(param_name):
if param_name in DEFAULTS:
return DEFAULTS[param_name]
else:
return None
def get_parameter(kwargs, singular_name, plural_name, use_default=True):
if singular_name in kwargs:
p = kwargs[singular_name]
else:
p = (flask.request.args.get(singular_name) or
flask.request.args.get(plural_name))
if p:
return p.split(',')
elif use_default:
default = get_default(singular_name)
return [default] if default else None
else:
return []
def record_filter(ignore=None, use_default=True):
if not ignore:
ignore = []
def decorator(f):
@functools.wraps(f)
def decorated_function(*args, **kwargs):
@ -58,34 +101,42 @@ def record_filter(parameter_getter=lambda x: flask.request.args.get(x)):
memory_storage = vault['memory_storage']
record_ids = memory_storage.get_record_ids()
param = parameter_getter('modules')
if param:
record_ids &= memory_storage.get_record_ids_by_modules(
param.split(','))
if 'module' not in ignore:
param = get_parameter(kwargs, 'module', 'modules', use_default)
if param:
record_ids &= (
memory_storage.get_record_ids_by_modules(param))
if 'launchpad_id' in kwargs:
param = kwargs['launchpad_id']
else:
param = (parameter_getter('launchpad_id') or
parameter_getter('launchpad_ids'))
if param:
record_ids &= memory_storage.get_record_ids_by_launchpad_ids(
param.split(','))
if 'project_type' not in ignore:
param = get_parameter(kwargs, 'project_type', 'project_types',
use_default)
if param:
modules = [module for module, project_type
in vault['modules'].iteritems()
if project_type in param]
record_ids &= (
memory_storage.get_record_ids_by_modules(modules))
if 'company' in kwargs:
param = kwargs['company']
else:
param = (parameter_getter('company') or
parameter_getter('companies'))
if param:
record_ids &= memory_storage.get_record_ids_by_companies(
param.split(','))
if 'launchpad_id' not in ignore:
param = get_parameter(kwargs, 'launchpad_id', 'launchpad_ids')
if param:
record_ids &= (
memory_storage.get_record_ids_by_launchpad_ids(param))
param = parameter_getter('release') or parameter_getter('releases')
if param:
if param != 'all':
record_ids &= memory_storage.get_record_ids_by_releases(
c.lower() for c in param.split(','))
if 'company' not in ignore:
param = get_parameter(kwargs, 'company', 'companies')
if param:
record_ids &= (
memory_storage.get_record_ids_by_companies(param))
if 'release' not in ignore:
param = get_parameter(kwargs, 'release', 'releases',
use_default)
if param:
if 'all' not in param:
record_ids &= (
memory_storage.get_record_ids_by_releases(
c.lower() for c in param))
kwargs['records'] = memory_storage.get_records(record_ids)
return f(*args, **kwargs)
@ -100,14 +151,15 @@ def aggregate_filter():
@functools.wraps(f)
def decorated_function(*args, **kwargs):
metric_filter = lambda r: r['loc']
metric_param = flask.request.args.get('metric')
if metric_param:
metric = metric_param.lower()
if metric == 'commits':
metric_filter = lambda r: 1
elif metric != 'loc':
raise Exception('Invalid metric %s' % metric)
metric_param = (flask.request.args.get('metric') or
DEFAULTS['metric'])
metric = metric_param.lower()
if metric == 'commits':
metric_filter = lambda r: 1
elif metric == 'loc':
metric_filter = lambda r: r['loc']
else:
raise Exception('Invalid metric %s' % metric)
kwargs['metric_filter'] = metric_filter
return f(*args, **kwargs)
@ -132,9 +184,11 @@ def exception_handler():
return decorator
DEFAULT_METRIC = 'loc'
DEFAULT_RELEASE = 'havana'
DEFAULT_PROJECT_TYPE = 'incubation'
DEFAULTS = {
'metric': 'commits',
'release': 'havana',
'project_type': 'openstack',
}
INDEPENDENT = '*independent'
@ -144,9 +198,8 @@ METRIC_LABELS = {
}
PROJECT_TYPES = {
'core': ['core'],
'incubation': ['core', 'incubation'],
'all': ['core', 'incubation', 'dev'],
'openstack': 'OpenStack',
'stackforge': 'StackForge',
}
ISSUE_TYPES = ['bug', 'blueprint']
@ -172,9 +225,15 @@ def templated(template=None):
metric = flask.request.args.get('metric')
if metric not in METRIC_LABELS:
metric = None
ctx['metric'] = metric or DEFAULT_METRIC
ctx['metric'] = metric or DEFAULTS['metric']
ctx['metric_label'] = METRIC_LABELS[ctx['metric']]
project_type = flask.request.args.get('project_type')
if project_type not in PROJECT_TYPES:
project_type = None
ctx['project_type'] = project_type or DEFAULTS['project_type']
ctx['project_type_label'] = PROJECT_TYPES[ctx['project_type']]
release = flask.request.args.get('release')
releases = vault['releases']
if release:
@ -183,7 +242,7 @@ def templated(template=None):
release = None
else:
release = releases[release]['release_name']
ctx['release'] = (release or DEFAULT_RELEASE).lower()
ctx['release'] = (release or DEFAULTS['release']).lower()
return flask.render_template(template_name, **ctx)
@ -329,22 +388,16 @@ def get_engineers(records, metric_filter):
@app.route('/data/timeline')
@exception_handler()
@record_filter(parameter_getter=lambda x: flask.request.args.get(x)
if (x != "release") and (x != "releases") else None)
def timeline(records):
@record_filter(ignore='release')
def timeline(records, **kwargs):
# find start and end dates
release_name = flask.request.args.get('release')
if not release_name:
release_name = DEFAULT_RELEASE
else:
release_name = release_name.lower()
release_names = get_parameter(kwargs, 'release', 'releases')
releases = get_vault()['releases']
if release_name not in releases:
if not release_names:
flask.abort(404)
release = releases[release_name]
if not (set(release_names) & set(releases.keys())):
flask.abort(404)
release = releases[release_names[0]]
start_date = release_start_date = user_utils.timestamp_to_week(
user_utils.date_to_timestamp(release['start_date']))
@ -365,6 +418,7 @@ def timeline(records):
weeks = range(start_date, end_date)
week_stat_loc = dict((c, 0) for c in weeks)
week_stat_commits = dict((c, 0) for c in weeks)
week_stat_commits_hl = dict((c, 0) for c in weeks)
# fill stats with the data
for record in records:
@ -372,6 +426,8 @@ def timeline(records):
if week in weeks:
week_stat_loc[week] += record['loc']
week_stat_commits[week] += 1
if 'all' in release_names or record['release'] in release_names:
week_stat_commits_hl[week] += 1
# form arrays in format acceptable to timeline plugin
array_loc = []
@ -381,9 +437,8 @@ def timeline(records):
for week in weeks:
week_str = user_utils.week_to_date(week)
array_loc.append([week_str, week_stat_loc[week]])
if release_start_date <= week <= release_end_date:
array_commits_hl.append([week_str, week_stat_commits[week]])
array_commits.append([week_str, week_stat_commits[week]])
array_commits_hl.append([week_str, week_stat_commits_hl[week]])
return json.dumps([array_commits, array_commits_hl, array_loc])
@ -408,37 +463,35 @@ def safe_encode(s):
@app.template_filter('link')
def make_link(title, uri=None):
param_names = ('release', 'metric', 'project_type')
param_values = {}
for param_name in param_names:
v = get_parameter({}, param_name, param_name)
if v:
param_values[param_name] = ','.join(v)
if param_values:
uri += '?' + '&'.join(['%s=%s' % (n, v)
for n, v in param_values.iteritems()])
return '<a href="%(uri)s">%(title)s</a>' % {'uri': uri, 'title': title}
def clear_text(s):
return cgi.escape(re.sub(r'\n{2,}', '\n', s, flags=re.MULTILINE))
def link_blueprint(s, module):
return re.sub(r'(blueprint\s+)([\w-]+)',
r'\1<a href="https://blueprints.launchpad.net/' +
module + r'/+spec/\2">\2</a>',
s, flags=re.IGNORECASE)
def link_bug(s):
return re.sub(r'(bug\s+)#?([\d]{5,7})',
r'\1<a href="https://bugs.launchpad.net/bugs/\2">\2</a>',
s, flags=re.IGNORECASE)
def link_change_id(s):
return re.sub(r'\s+(I[0-9a-f]{40})',
r' <a href="https://review.openstack.org/#q,\1,n,z">\1</a>',
s)
@app.template_filter('commit_message')
def make_commit_message(record):
s = record['message']
module = record['module']
return link_change_id(link_bug(link_blueprint(clear_text(
record['message']), record['module'])))
# clear text
s = cgi.escape(re.sub(re.compile('\n{2,}', flags=re.MULTILINE), '\n', s))
# insert links
s = re.sub(re.compile('(blueprint\s+)([\w-]+)', flags=re.IGNORECASE),
r'\1<a href="https://blueprints.launchpad.net/' +
module + r'/+spec/\2">\2</a>', s)
s = re.sub(re.compile('(bug\s+)#?([\d]{5,7})', flags=re.IGNORECASE),
r'\1<a href="https://bugs.launchpad.net/bugs/\2">\2</a>', s)
s = re.sub(r'\s+(I[0-9a-f]{40})',
r' <a href="https://review.openstack.org/#q,\1,n,z">\1</a>', s)
return s
gravatar = gravatar_ext.Gravatar(app,

89
etc/default_data.json
View File

@ -4,6 +4,19 @@
},
"users": [
{
"launchpad_id": "openstack",
"companies": [
{
"company_name": "*robots",
"end_date": null
}
],
"user_name": "OpenStack Robot",
"emails": [
"review@openstack.org", "jenkins@review.openstack.org", "jenkins@openstack.org", "hudson@openstack.org"
]
},
{
"launchpad_id": "akamyshnikova",
"companies": [
@ -12661,6 +12674,10 @@
"companies": [
{
"company_name": "Rackspace",
"end_date": "2012-Nov-01"
},
{
"company_name": "SwiftStack",
"end_date": null
}
],
@ -13429,6 +13446,10 @@
"domains": ["vexxhost.com"],
"company_name": "VexxHost"
},
{
"domains": ["vbridges.com"],
"company_name": "VirtualBridges"
},
{
"domains": ["virtualtech.jp"],
"company_name": "Virtualtech"
@ -13454,8 +13475,8 @@
"repos": [
{
"branches": ["master"],
"name": "Neutron Client",
"type": "core",
"module": "python-neutronclient",
"project_type": "openstack",
"uri": "git://github.com/openstack/python-neutronclient.git",
"releases": [
{
@ -13477,8 +13498,8 @@
},
{
"branches": ["master"],
"name": "Keystone",
"type": "core",
"module": "keystone",
"project_type": "openstack",
"uri": "git://github.com/openstack/keystone.git",
"releases": [
{
@ -13505,8 +13526,64 @@
},
{
"branches": ["master"],
"name": "Murano API",
"type": "stackforge",
"module": "nova",
"project_type": "openstack",
"uri": "git://github.com/openstack/nova.git",
"releases": [
{
"release_name": "Essex",
"tag_from": "2011.3",
"tag_to": "2012.1"
},
{
"release_name": "Folsom",
"tag_from": "2012.1",
"tag_to": "2012.2"
},
{
"release_name": "Grizzly",
"tag_from": "2012.2",
"tag_to": "2013.1"
},
{
"release_name": "Havana",
"tag_from": "2013.1",
"tag_to": "HEAD"
}
]
},
{
"branches": ["master"],
"module": "neutron",
"project_type": "openstack",
"uri": "git://github.com/openstack/neutron.git",
"releases": [
{
"release_name": "Essex",
"tag_from": "2011.3",
"tag_to": "2012.1"
},
{
"release_name": "Folsom",
"tag_from": "2012.1",
"tag_to": "2012.2"
},
{
"release_name": "Grizzly",
"tag_from": "2012.2",
"tag_to": "2013.1"
},
{
"release_name": "Havana",
"tag_from": "2013.1",
"tag_to": "HEAD"
}
]
},
{
"branches": ["master"],
"module": "murano-api",
"project_type": "stackforge",
"uri": "git://github.com/stackforge/murano-api.git",
"releases": [
{

4
etc/test_default_data.json
View File

@ -39,7 +39,7 @@
"repos": [
{
"branches": ["master"],
"name": "Quantum Client",
"module": "python-quantumclient",
"type": "core",
"uri": "git://github.com/openstack/python-quantumclient.git",
"releases": [
@ -62,7 +62,7 @@
},
{
"branches": ["master"],
"name": "Keystone",
"module": "keystone",
"type": "core",
"uri": "git://github.com/openstack/keystone.git",
"releases": [

66
tests/unit/test_web_utils.py Normal file
View File

@ -0,0 +1,66 @@
import testtools
from dashboard import web
class TestWebUtils(testtools.TestCase):
def setUp(self):
super(TestWebUtils, self).setUp()
def test_make_commit_message(self):
message = '''
During finish_migration the manager calls initialize_connection but doesn't
update the block_device_mapping with the potentially new connection_info
returned.
Fixes bug 1076801
Change-Id: Ie49ccd2138905e178843b375a9b16c3fe572d1db'''
module = 'test'
record = {
'message': message,
'module': module,
}
expected = '''
During finish_migration the manager calls initialize_connection but doesn't
update the block_device_mapping with the potentially new connection_info
returned.
Fixes bug <a href="https://bugs.launchpad.net/bugs/1076801">1076801</a>
''' + ('Change-Id: <a href="https://review.openstack.org/#q,'
'Ie49ccd2138905e178843b375a9b16c3fe572d1db,n,z">'
'Ie49ccd2138905e178843b375a9b16c3fe572d1db</a>')
observed = web.make_commit_message(record)
self.assertEqual(expected, observed,
'Commit message should be processed correctly')
def test_make_commit_message_blueprint_link(self):
message = '''
Implemented new driver for Cinder <:
Implements Blueprint super-driver
Change-Id: Ie49ccd2138905e178843b375a9b16c3fe572d1db'''
module = 'cinder'
record = {
'message': message,
'module': module,
}
expected = '''
Implemented new driver for Cinder &lt;:
Implements Blueprint ''' + (
'<a href="https://blueprints.launchpad.net/cinder/+spec/'
'super-driver">super-driver</a>' + '\n' +
'Change-Id: <a href="https://review.openstack.org/#q,'
'Ie49ccd2138905e178843b375a9b16c3fe572d1db,n,z">'
'Ie49ccd2138905e178843b375a9b16c3fe572d1db</a>')
observed = web.make_commit_message(record)
self.assertEqual(expected, observed,
'Commit message should be processed correctly')