Implementation of blueprint stackalytics-core
* Data updater is implemented * Completed implementation of commit processor * Logging is added into commit processor and runtime storage * Commit processor is fixed * Domain-company map is inverted * Extracted get update count into separate function * Fixed regex that matches diff statistics (lines inserted, lines deleted and files changed) * Implemented caching of unknown users * Replaced dictionaries by sets for pids and branches * Vcs is responsible for module and branches fields of commit record * Added release tags support * Implemented statistics by company * Added config for releases * Implemented front-end for companies details * Implemented front-end for modules details * Fixed metric switch * Implemented timeline rendering * Release selector is fixed * Chdir is needed after cloning a new repo * Company details screen is implemented * Fixed invalid emails processing by Launchpad * Fixed parsing of 0 files changed case * Module details screen implemented * Commit message is cleared and links are inserted * Engineer details screen is implemented * Fixed mapping from company to email for subdomains of 3rd level * Fixed wrong user structure for users not found by LP * Also coverage for commit processor * Fixed company matching algorithm * The company was not matched when user email had more domains than company's one * Add option to enforce sync with default data * Default data is added. Old confs removed * Add *.local into gitignore Scripts cleanup Moved from pylibmc to python-memcached Library pylibmc depends on libmemcached and doesn't work on CentOS (version conflict bw lib requirement and memcached). Change-Id: I0cc61c6d344ba24442ec954635010b518c0efa95
This commit is contained in:
Ilya Shakhat
parent
a5f1411218
commit
b7f19335f6
1
.gitignore
vendored
1
.gitignore
vendored
@ -1,5 +1,6 @@
|
||||
*~
|
||||
*.pyc
|
||||
*.local
|
||||
AUTHORS
|
||||
ChangeLog
|
||||
MANIFEST
|
||||
|
||||
@ -5,7 +5,7 @@ import os
|
||||
import sys
|
||||
sys.path.insert(0, os.getcwd())
|
||||
|
||||
from dashboard.dashboard import app
|
||||
from dashboard.web import app
|
||||
|
||||
|
||||
app.run()
|
||||
@ -1,31 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [[ -z $STACKALYTICS_HOME ]]; then
|
||||
echo "Variable STACKALYTICS_HOME must be specified"
|
||||
exit
|
||||
fi
|
||||
|
||||
echo "Analytics home is $STACKALYTICS_HOME"
|
||||
|
||||
DASHBOARD_CONF='$STACKALYTICS_HOME/conf/dashboard.conf'
|
||||
|
||||
TOP_DIR=$(cd $(dirname "$0") && pwd)
|
||||
|
||||
DB_FILE=`mktemp -u --tmpdir=$STACKALYTICS_HOME/data stackalytics-XXXXXXXXXXXX.sqlite`
|
||||
TEMP_CONF=`mktemp -u`
|
||||
|
||||
cd $TOP_DIR/../scripts/
|
||||
./pull-repos.sh
|
||||
|
||||
cd $TOP_DIR/../
|
||||
./bin/stackalytics --config-file $STACKALYTICS_HOME/conf/analytics.conf --db-database $DB_FILE --verbose
|
||||
|
||||
DATE=`date -u +'%d-%b-%y %H:%M %Z'`
|
||||
|
||||
echo DATABASE = \'$DB_FILE\' >> $TEMP_CONF
|
||||
echo LAST_UPDATE = \'$DATE\' >> $TEMP_CONF
|
||||
|
||||
#rm $DASHBOARD_CONF
|
||||
#mv $TEMP_CONF $DASHBOARD_CONF
|
||||
|
||||
echo "Data is refreshed, please restart service"
|
||||
@ -1,5 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
TOP_DIR=$(cd $(dirname "$0") && pwd)
|
||||
|
||||
./tools/with_venv.sh python scripts/launchpad/grab-unmapped-launchpad-ids.py
|
||||
@ -5,7 +5,7 @@ import os
|
||||
import sys
|
||||
sys.path.insert(0, os.getcwd())
|
||||
|
||||
from pycvsanaly2.main import main
|
||||
from stackalytics.processor.main import main
|
||||
|
||||
|
||||
main()
|
||||
18
bin/update
18
bin/update
@ -1,18 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [[ -z $STACKALYTICS_HOME ]]; then
|
||||
echo "Variable STACKALYTICS_HOME must be specified"
|
||||
exit
|
||||
fi
|
||||
|
||||
echo "Analytics home is $STACKALYTICS_HOME"
|
||||
|
||||
CONF="$STACKALYTICS_HOME/conf/analytics.conf"
|
||||
|
||||
TOP_DIR=$(cd $(dirname "$0") && pwd)
|
||||
cd $TOP_DIR/../scripts/
|
||||
./pull-repos.sh
|
||||
|
||||
echo "Updating data"
|
||||
cd $TOP_DIR/../
|
||||
./bin/stackalytics --config-file $CONF --db-database $STACKALYTICS_HOME/data/stackalyticss.sqlite --verbose
|
||||
100
dashboard/memory_storage.py
Normal file
100
dashboard/memory_storage.py
Normal file
@ -0,0 +1,100 @@
|
||||
from stackalytics.processor import user_utils
|
||||
|
||||
MEMORY_STORAGE_CACHED = 0
|
||||
|
||||
|
||||
class MemoryStorage(object):
|
||||
def __init__(self, records):
|
||||
pass
|
||||
|
||||
|
||||
class CachedMemoryStorage(MemoryStorage):
|
||||
def __init__(self, records):
|
||||
super(CachedMemoryStorage, self).__init__(records)
|
||||
|
||||
self.records = {}
|
||||
self.company_index = {}
|
||||
self.date_index = {}
|
||||
self.module_index = {}
|
||||
self.launchpad_id_index = {}
|
||||
self.release_index = {}
|
||||
self.dates = []
|
||||
for record in records:
|
||||
self.records[record['record_id']] = record
|
||||
self.index(record)
|
||||
self.dates = sorted(self.date_index)
|
||||
self.company_name_mapping = dict((c.lower(), c)
|
||||
for c in self.company_index.keys())
|
||||
|
||||
def index(self, record):
|
||||
|
||||
self._add_to_index(self.company_index, record, 'company_name')
|
||||
self._add_to_index(self.module_index, record, 'module')
|
||||
self._add_to_index(self.launchpad_id_index, record, 'launchpad_id')
|
||||
self._add_to_index(self.release_index, record, 'release')
|
||||
self._add_to_index(self.date_index, record, 'date')
|
||||
|
||||
record['week'] = user_utils.timestamp_to_week(record['date'])
|
||||
record['loc'] = record['lines_added'] + record['lines_deleted']
|
||||
|
||||
def _add_to_index(self, record_index, record, key):
|
||||
record_key = record[key]
|
||||
if record_key in record_index:
|
||||
record_index[record_key].add(record['record_id'])
|
||||
else:
|
||||
record_index[record_key] = set([record['record_id']])
|
||||
|
||||
def _get_record_ids_from_index(self, items, index):
|
||||
record_ids = set()
|
||||
for item in items:
|
||||
if item not in index:
|
||||
raise Exception('Parameter %s not valid' % item)
|
||||
record_ids |= index[item]
|
||||
return record_ids
|
||||
|
||||
def get_record_ids_by_modules(self, modules):
|
||||
return self._get_record_ids_from_index(modules, self.module_index)
|
||||
|
||||
def get_record_ids_by_companies(self, companies):
|
||||
return self._get_record_ids_from_index(
|
||||
map(self._get_company_name, companies),
|
||||
self.company_index)
|
||||
|
||||
def get_record_ids_by_launchpad_ids(self, launchpad_ids):
|
||||
return self._get_record_ids_from_index(launchpad_ids,
|
||||
self.launchpad_id_index)
|
||||
|
||||
def get_record_ids_by_releases(self, releases):
|
||||
return self._get_record_ids_from_index(releases, self.release_index)
|
||||
|
||||
def get_record_ids(self):
|
||||
return set(self.records.keys())
|
||||
|
||||
def get_records(self, record_ids):
|
||||
for i in record_ids:
|
||||
yield self.records[i]
|
||||
|
||||
def _get_company_name(self, company_name):
|
||||
normalized = company_name.lower()
|
||||
if normalized not in self.company_name_mapping:
|
||||
raise Exception('Unknown company name %s' % company_name)
|
||||
return self.company_name_mapping[normalized]
|
||||
|
||||
def get_companies(self):
|
||||
return self.company_index.keys()
|
||||
|
||||
def get_modules(self):
|
||||
return self.module_index.keys()
|
||||
|
||||
def get_launchpad_ids(self):
|
||||
return self.launchpad_id_index.keys()
|
||||
|
||||
|
||||
class MemoryStorageFactory(object):
|
||||
|
||||
@staticmethod
|
||||
def get_storage(memory_storage_type, records):
|
||||
if memory_storage_type == MEMORY_STORAGE_CACHED:
|
||||
return CachedMemoryStorage(records)
|
||||
else:
|
||||
raise Exception('Unknown memory storage type')
|
||||
@ -94,7 +94,7 @@ div.drops label {
|
||||
color: #909cb5;
|
||||
}
|
||||
|
||||
span.drop_period {
|
||||
span.drop_release {
|
||||
margin-top: 6px;
|
||||
display: block;
|
||||
height: 30px;
|
||||
|
||||
@ -34,9 +34,9 @@
|
||||
<link rel='archives' title='July 2011' href='http://www.mirantis.com/2011/07/' />
|
||||
<link rel='archives' title='June 2011' href='http://www.mirantis.com/2011/06/' />
|
||||
<link rel='archives' title='May 2011' href='http://www.mirantis.com/2011/05/' />
|
||||
<link href='http://fonts.googleapis.com/css?family=PT+Sans:400,700,400italic&subset=latin,cyrillic' rel='stylesheet' type='text/css'>
|
||||
<link href='http://fonts.googleapis.com/css?family=PT+Sans+Caption&subset=latin,cyrillic' rel='stylesheet' type='text/css'>
|
||||
<link href='http://fonts.googleapis.com/css?family=PT+Sans+Narrow:400,700&subset=latin,cyrillic' rel='stylesheet' type='text/css'>
|
||||
<link href='http://fonts.googleapis.com/css?family=PT+Sans:400,700,400italic&subset=latin,cyrillic' rel='stylesheet' type='text/css' />
|
||||
<link href='http://fonts.googleapis.com/css?family=PT+Sans+Caption&subset=latin,cyrillic' rel='stylesheet' type='text/css' />
|
||||
<link href='http://fonts.googleapis.com/css?family=PT+Sans+Narrow:400,700&subset=latin,cyrillic' rel='stylesheet' type='text/css' />
|
||||
|
||||
|
||||
{# <script type="text/javascript" src="{{ url_for('static', filename='js/jquery-1.9.1.min.js') }}"></script>#}
|
||||
@ -49,21 +49,21 @@
|
||||
{% block head %}{% endblock %}
|
||||
|
||||
<!-- Google Analytics -->
|
||||
<script type="text/javascript">
|
||||
var _gaq = _gaq || [];
|
||||
_gaq.push(['_setAccount', 'UA-8933515-2']);
|
||||
_gaq.push(['_setDomainName', 'stackalytics.com']);
|
||||
_gaq.push(['_setAllowLinker', true]);
|
||||
_gaq.push(['_trackPageview']);
|
||||
(function () {
|
||||
var ga = document.createElement('script');
|
||||
ga.type = 'text/javascript';
|
||||
ga.async = true;
|
||||
ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
|
||||
var s = document.getElementsByTagName('script')[0];
|
||||
s.parentNode.insertBefore(ga, s);
|
||||
})();
|
||||
</script>
|
||||
{# <script type="text/javascript">#}
|
||||
{# var _gaq = _gaq || [];#}
|
||||
{# _gaq.push(['_setAccount', 'UA-8933515-2']);#}
|
||||
{# _gaq.push(['_setDomainName', 'stackalytics.com']);#}
|
||||
{# _gaq.push(['_setAllowLinker', true]);#}
|
||||
{# _gaq.push(['_trackPageview']);#}
|
||||
{# (function () {#}
|
||||
{# var ga = document.createElement('script');#}
|
||||
{# ga.type = 'text/javascript';#}
|
||||
{# ga.async = true;#}
|
||||
{# ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';#}
|
||||
{# var s = document.getElementsByTagName('script')[0];#}
|
||||
{# s.parentNode.insertBefore(ga, s);#}
|
||||
{# })();#}
|
||||
{# </script>#}
|
||||
|
||||
</head>
|
||||
|
||||
|
||||
@ -4,16 +4,19 @@
|
||||
{{ company }}
|
||||
{% endblock %}
|
||||
|
||||
{% block scripts %}
|
||||
<script type="text/javascript">
|
||||
chartAndTableRenderer("/data/engineers", "left_list", "left_chart", "/engineers/", {company: "{{ company|encode }}" });
|
||||
chartAndTableRenderer("/data/modules", "right_list", "right_chart", "/modules/", {company: "{{ company|encode }}" });
|
||||
timelineRenderer({company: "{{ company|encode }}" })
|
||||
</script>
|
||||
{% endblock %}
|
||||
|
||||
|
||||
{% block left_frame %}
|
||||
|
||||
<h2>Contribution by engineers</h2>
|
||||
|
||||
<script type="text/javascript">
|
||||
loadTable("left_list", "/data/companies/{{ company|encode }}");
|
||||
loadChart("left_chart", "/data/companies/{{ company|encode }}", {limit: 10});
|
||||
loadTimeline({company: '{{ company }}'})
|
||||
</script>
|
||||
|
||||
<div id="left_chart" style="width: 100%; height: 350px;"></div>
|
||||
|
||||
<table id="left_list" class="display">
|
||||
@ -35,11 +38,6 @@
|
||||
|
||||
<h2>Contribution by modules</h2>
|
||||
|
||||
<script type="text/javascript">
|
||||
loadTable("right_list", "/data/modules", {company: "{{ company|encode }}" });
|
||||
loadChart("right_chart", "/data/modules", {limit: 10, company: "{{ company|encode }}" });
|
||||
</script>
|
||||
|
||||
<div id="right_chart" style="width: 100%; height: 350px;"></div>
|
||||
|
||||
<table id="right_list" class="display">
|
||||
@ -59,10 +57,10 @@
|
||||
{% if blueprints %}
|
||||
<div>Blueprints:
|
||||
<ol>
|
||||
{% for one in blueprints %}
|
||||
{% for rec in blueprints %}
|
||||
<li>
|
||||
<a href="https://blueprints.launchpad.net/{{ one[1] }}/+spec/{{ one[0] }}">{{ one[0] }}</a>
|
||||
<small>{{ one[1] }}</small>
|
||||
<a href="https://blueprints.launchpad.net/{{ rec['module'] }}/+spec/{{ rec['id'] }}">{{ rec['id'] }}</a>
|
||||
<small>{{ rec['module'] }}</small>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ol>
|
||||
@ -70,12 +68,18 @@
|
||||
{% endif %}
|
||||
|
||||
{% if bugs %}
|
||||
<div>Bug fixes: <b>{{ bugs|length }}</b>
|
||||
<div>Bugs:
|
||||
<ol>
|
||||
{% for rec in bugs %}
|
||||
<li>
|
||||
<a href="https://bugs.launchpad.net/bugs/{{ rec['id'] }}">{{ rec['id'] }}</a>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ol>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<div>Total commits: <b>{{ commits|length }}</b>, among them <b>{{ code_commits }}</b> commits in code,
|
||||
among them <b>{{ test_only_commits }}</b> test only commits</div>
|
||||
<div>Total commits: <b>{{ commits|length }}</b></div>
|
||||
<div>Total LOC: <b>{{ loc }}</b></div>
|
||||
|
||||
{% endblock %}
|
||||
|
||||
@ -1,25 +1,26 @@
|
||||
{% extends "layout.html" %}
|
||||
|
||||
{% block title %}
|
||||
{{ details.name }}
|
||||
{{ user.user_name }}
|
||||
{% endblock %}
|
||||
|
||||
{% block scripts %}
|
||||
<script type="text/javascript">
|
||||
chartAndTableRenderer("/data/modules", "right_list", "right_chart", "/modules/", {launchpad_id: "{{ launchpad_id }}" });
|
||||
timelineRenderer({launchpad_id: "{{ launchpad_id }}" })
|
||||
</script>
|
||||
{% endblock %}
|
||||
|
||||
{% block left_frame %}
|
||||
|
||||
<script type="text/javascript">
|
||||
loadTimeline({engineer: '{{engineer}}'})
|
||||
</script>
|
||||
|
||||
<div style='float: left;'>
|
||||
<img src="{{ details.email|gravatar(size=64) }}">
|
||||
<img src="{{ user.emails[0]|gravatar(size=64) }}">
|
||||
</div>
|
||||
<div style='margin-left: 90px;'>
|
||||
<h2 style='margin-bottom: 0.5em;'>{{ details.name }}</h2>
|
||||
{% if details.company %}
|
||||
<div>Company: {{ link('/companies/' + details.company, details.company) }}</div>
|
||||
<h2 style='margin-bottom: 0.5em;'>{{ user.user_name }}</h2>
|
||||
{% if user.companies %}
|
||||
<div>Company: {{ user.companies[-1].company_name|link('/companies/' + user.companies[-1].company_name)|safe }}</div>
|
||||
{% endif %}
|
||||
<div>Launchpad: <a href="https://launchpad.net/~{{ details.launchpad_id }}">{{ details.launchpad_id }}</a></div>
|
||||
{# <div>Email: {{ details.email }}</div>#}
|
||||
<div>Launchpad: <a href="https://launchpad.net/~{{ launchpad_id }}">{{ launchpad_id }}</a></div>
|
||||
</div>
|
||||
|
||||
<h3>Commits history</h3>
|
||||
@ -28,15 +29,13 @@
|
||||
<div>There are no commits for selected period or project type.</div>
|
||||
{% endif %}
|
||||
|
||||
{% for message in commits %}
|
||||
{% for rec in commits %}
|
||||
<div>
|
||||
<h4>{{ message.date|datetimeformat }} to <a href="https://launchpad.net/{{ message.module }}">{{ message.module }}</a>
|
||||
{% if message.is_code %} <span style="color: royalblue">code</span> {% endif %}
|
||||
{% if message.is_test %} <span style="color: magenta">test</span> {% endif %}
|
||||
<h4>{{ rec.date|datetimeformat }} to <a href="https://launchpad.net/{{ rec.module }}">{{ rec.module }}</a>
|
||||
</h4>
|
||||
<div style='white-space: pre-wrap; padding-left: 2em;'>{{ message.message|safe }}</div>
|
||||
<div style="padding-left: 2em;"><span style="color: green">+ {{ message.added_loc }}</span>
|
||||
<span style="color: red">- {{ message.removed_loc }}</span></div>
|
||||
<div style='white-space: pre-wrap; padding-left: 2em;'>{{ rec|commit_message|safe }}</div>
|
||||
<div style="padding-left: 2em;"><span style="color: green">+ {{ rec.lines_added }}</span>
|
||||
<span style="color: red">- {{ rec.lines_deleted }}</span></div>
|
||||
</div>
|
||||
{% endfor %}
|
||||
|
||||
@ -47,11 +46,6 @@
|
||||
{% if commits %}
|
||||
<h2>Contribution by modules</h2>
|
||||
|
||||
<script type="text/javascript">
|
||||
loadTable("right_list", "/data/modules", {engineer: "{{ engineer }}" });
|
||||
loadChart("right_chart", "/data/modules", {limit: 10, engineer: "{{ engineer }}" });
|
||||
</script>
|
||||
|
||||
<div id="right_chart" style="width: 100%; height: 350px;"></div>
|
||||
|
||||
<table id="right_list" class="display">
|
||||
@ -71,10 +65,10 @@
|
||||
{% if blueprints %}
|
||||
<div>Blueprints:
|
||||
<ol>
|
||||
{% for one in blueprints %}
|
||||
{% for rec in blueprints %}
|
||||
<li>
|
||||
<a href="https://blueprints.launchpad.net/{{ one[1] }}/+spec/{{ one[0] }}">{{ one[0] }}</a>
|
||||
<small>{{ one[1] }}</small>
|
||||
<a href="https://blueprints.launchpad.net/{{ rec['module'] }}/+spec/{{ rec['id'] }}">{{ rec['id'] }}</a>
|
||||
<small>{{ rec['module'] }}</small>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ol>
|
||||
@ -82,23 +76,18 @@
|
||||
{% endif %}
|
||||
|
||||
{% if bugs %}
|
||||
<div>Bug fixes:
|
||||
<div>Bugs:
|
||||
<ol>
|
||||
{% for one in bugs %}
|
||||
{% for rec in bugs %}
|
||||
<li>
|
||||
<a href="https://bugs.launchpad.net/bugs/{{ one[0] }}">{{ one[0] }}</a>
|
||||
<small>
|
||||
{% if one[1] %} <span style="color: royalblue">C</span> {% endif %}
|
||||
{% if one[2] %} <span style="color: magenta">T</span> {% endif %}
|
||||
</small>
|
||||
<a href="https://bugs.launchpad.net/bugs/{{ rec['id'] }}">{{ rec['id'] }}</a>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ol>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<div>Total commits: <b>{{ commits|length }}</b>, among them <b>{{ code_commits }}</b> commits in code,
|
||||
among them <b>{{ test_only_commits }}</b> test only commits</div>
|
||||
<div>Total commits: <b>{{ commits|length }}</b></div>
|
||||
<div>Total LOC: <b>{{ loc }}</b></div>
|
||||
{% endif %}
|
||||
|
||||
|
||||
@ -18,159 +18,173 @@
|
||||
<script type="text/javascript" src="{{ url_for('static', filename='js/jqplot.highlighter.min.js') }}"></script>
|
||||
|
||||
<script type="text/javascript">
|
||||
// load table data
|
||||
function loadTable(table_id, source, options) {
|
||||
|
||||
function showTimeline(data) {
|
||||
var plot = $.jqplot('timeline', data, {
|
||||
gridPadding: {
|
||||
right: 35
|
||||
},
|
||||
cursor: {
|
||||
show: false
|
||||
},
|
||||
highlighter: {
|
||||
show: true,
|
||||
sizeAdjust: 6
|
||||
},
|
||||
axes: {
|
||||
xaxis: {
|
||||
tickRenderer: $.jqplot.CanvasAxisTickRenderer,
|
||||
tickOptions: {
|
||||
fontSize: '8pt',
|
||||
angle: -90,
|
||||
formatString: '%b \'%y'
|
||||
},
|
||||
renderer: $.jqplot.DateAxisRenderer,
|
||||
tickInterval: '1 month'
|
||||
},
|
||||
yaxis: {
|
||||
min: 0,
|
||||
label: ''
|
||||
},
|
||||
y2axis: {
|
||||
min: 0,
|
||||
label: ''
|
||||
}
|
||||
},
|
||||
series: [
|
||||
{
|
||||
shadow: false,
|
||||
fill: true,
|
||||
fillColor: '#4bb2c5',
|
||||
fillAlpha: 0.3
|
||||
},
|
||||
{
|
||||
shadow: false,
|
||||
fill: true,
|
||||
color: '#4bb2c5',
|
||||
fillColor: '#4bb2c5'
|
||||
},
|
||||
{
|
||||
shadow: false,
|
||||
lineWidth: 1.5,
|
||||
showMarker: true,
|
||||
markerOptions: { size: 5 },
|
||||
yaxis: 'y2axis'
|
||||
}
|
||||
]
|
||||
});
|
||||
}
|
||||
|
||||
function timelineRenderer(options) {
|
||||
$(document).ready(function () {
|
||||
$("#"+table_id).dataTable({
|
||||
"aLengthMenu": [[25, 50, -1], [25, 50, "All"]],
|
||||
"aaSorting": [[ 2, "desc" ]],
|
||||
"bProcessing": true,
|
||||
"sAjaxSource": make_uri(source, options),
|
||||
"sPaginationType": "full_numbers",
|
||||
"iDisplayLength": 25,
|
||||
"aoColumns": [
|
||||
{ "mData": "index" },
|
||||
{ "mData": "link" },
|
||||
{ "mData": "rank" }
|
||||
]
|
||||
$.ajax({
|
||||
url: make_uri("/data/timeline", options),
|
||||
dataType: "json",
|
||||
success: function (data) {
|
||||
showTimeline(data);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// load chart
|
||||
function loadChart(chart_id, source, options) {
|
||||
function chartAndTableRenderer(url, table_id, chart_id, link_prefix, options) {
|
||||
|
||||
$(document).ready(function () {
|
||||
// Our ajax data renderer which here retrieves a text file.
|
||||
// it could contact any source and pull data, however.
|
||||
// The options argument isn't used in this renderer.
|
||||
var ajaxDataRenderer = function (url, plot, options) {
|
||||
var ret = null;
|
||||
$.ajax({
|
||||
// have to use synchronous here, else the function
|
||||
// will return before the data is fetched
|
||||
async: false,
|
||||
url: url,
|
||||
dataType: "json",
|
||||
success: function (data) {
|
||||
var array = [];
|
||||
for(i = 0; i < data['aaData'].length; i++) {
|
||||
array.push([data['aaData'][i].name, data['aaData'][i].rank]);
|
||||
|
||||
$.ajax({
|
||||
url: make_uri(url, options),
|
||||
dataType: "json",
|
||||
success: function (data) {
|
||||
|
||||
var tableData = [];
|
||||
var chartData = [];
|
||||
|
||||
var limit = 10;
|
||||
var aggregate = 0;
|
||||
var index = 1;
|
||||
var i;
|
||||
|
||||
for (i = 0; i < data.length; i++) {
|
||||
if (i < limit - 1) {
|
||||
chartData.push([data[i].name, data[i].metric]);
|
||||
} else {
|
||||
aggregate += data[i].metric;
|
||||
}
|
||||
ret = [array]
|
||||
}
|
||||
});
|
||||
return ret;
|
||||
};
|
||||
|
||||
// passing in the url string as the jqPlot data argument is a handy
|
||||
// shortcut for our renderer. You could also have used the
|
||||
// "dataRendererOptions" option to pass in the url.
|
||||
var plot = $.jqplot(chart_id, make_uri(source, options), {
|
||||
dataRenderer: ajaxDataRenderer,
|
||||
seriesDefaults: {
|
||||
// Make this a pie chart.
|
||||
renderer: jQuery.jqplot.PieRenderer,
|
||||
rendererOptions: {
|
||||
// Put data labels on the pie slices.
|
||||
// By default, labels show the percentage of the slice.
|
||||
showDataLabels: true
|
||||
var index_label = index;
|
||||
if (data[i].name == "*independent") {
|
||||
index_label = "";
|
||||
} else {
|
||||
index++;
|
||||
}
|
||||
var link = make_link(link_prefix, data[i].id, data[i].name);
|
||||
tableData.push({"index": index_label, "link": link, "metric": data[i].metric});
|
||||
}
|
||||
},
|
||||
legend: { show: true, location: 'e' }
|
||||
|
||||
if (i == limit) {
|
||||
chartData.push([data[i-1].name, data[i-1].metric]);
|
||||
} else if (i > limit) {
|
||||
chartData.push(["others", aggregate]);
|
||||
}
|
||||
|
||||
$("#" + table_id).dataTable({
|
||||
"aLengthMenu": [
|
||||
[25, 50, -1],
|
||||
[25, 50, "All"]
|
||||
],
|
||||
"aaSorting": [
|
||||
[ 2, "desc" ]
|
||||
],
|
||||
"sPaginationType": "full_numbers",
|
||||
"iDisplayLength": 25,
|
||||
"aaData": tableData,
|
||||
"aoColumns": [
|
||||
{ "mData": "index" },
|
||||
{ "mData": "link" },
|
||||
{ "mData": "metric" }
|
||||
]
|
||||
});
|
||||
|
||||
var plot = $.jqplot(chart_id, [chartData], {
|
||||
seriesDefaults: {
|
||||
renderer: jQuery.jqplot.PieRenderer,
|
||||
rendererOptions: {
|
||||
showDataLabels: true
|
||||
}
|
||||
},
|
||||
legend: { show: true, location: 'e' }
|
||||
});
|
||||
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// load timeline
|
||||
function loadTimeline(options) {
|
||||
$(document).ready(function () {
|
||||
var ajaxDataRenderer = function (url, plot, options) {
|
||||
var ret = null;
|
||||
$.ajax({
|
||||
// have to use synchronous here, else the function
|
||||
// will return before the data is fetched
|
||||
async: false,
|
||||
url: url,
|
||||
dataType: "json",
|
||||
success: function (data) {
|
||||
ret = data;
|
||||
}
|
||||
});
|
||||
return ret;
|
||||
};
|
||||
|
||||
var jsonurl = make_uri("/data/timeline", options);
|
||||
|
||||
var plot = $.jqplot('timeline', jsonurl, {
|
||||
dataRenderer: ajaxDataRenderer,
|
||||
dataRendererOptions: {
|
||||
unusedOptionalUrl: jsonurl
|
||||
},
|
||||
gridPadding: {right: 35},
|
||||
cursor: {
|
||||
show: false
|
||||
},
|
||||
highlighter: {
|
||||
show: true,
|
||||
sizeAdjust: 6
|
||||
},
|
||||
axes: {
|
||||
xaxis: {
|
||||
tickRenderer: $.jqplot.CanvasAxisTickRenderer,
|
||||
tickOptions: {
|
||||
fontSize: '8pt',
|
||||
angle: -90,
|
||||
formatString: '%b \'%y'
|
||||
},
|
||||
renderer: $.jqplot.DateAxisRenderer,
|
||||
tickInterval: '1 month'
|
||||
},
|
||||
yaxis: {
|
||||
min: 0,
|
||||
label: ''
|
||||
},
|
||||
y2axis: {
|
||||
min: 0,
|
||||
label: ''
|
||||
}
|
||||
},
|
||||
series: [
|
||||
{
|
||||
shadow: false,
|
||||
fill: true,
|
||||
fillColor: '#4bb2c5',
|
||||
fillAlpha: 0.3
|
||||
},
|
||||
{
|
||||
shadow: false,
|
||||
fill: true,
|
||||
color: '#4bb2c5',
|
||||
fillColor: '#4bb2c5'
|
||||
},
|
||||
{
|
||||
shadow: false,
|
||||
lineWidth: 1.5,
|
||||
showMarker: true,
|
||||
markerOptions: { size: 5 },
|
||||
yaxis: 'y2axis'
|
||||
}
|
||||
]
|
||||
});
|
||||
function getUrlVars() {
|
||||
var vars = {};
|
||||
var parts = window.location.href.replace(/[?&]+([^=&]+)=([^&]*)/gi, function (m, key, value) {
|
||||
vars[key] = value;
|
||||
});
|
||||
return vars;
|
||||
}
|
||||
|
||||
$(document).ready(function () {
|
||||
$('#metric').val('{{ metric }}');
|
||||
$('#period').val('{{ period }}');
|
||||
$('#release').val('{{ release }}');
|
||||
$('#project_type').val('{{ project_type }}');
|
||||
});
|
||||
|
||||
function make_link(uri_prefix, id, title, options) {
|
||||
var link = make_uri(uri_prefix + encodeURIComponent(id).toLowerCase(), options);
|
||||
return "<a href=\"" + link + "\">" + title + "</a>"
|
||||
}
|
||||
|
||||
function make_uri(uri, options) {
|
||||
var ops = {};
|
||||
if (options != null) {
|
||||
$.extend(ops, options);
|
||||
}
|
||||
$.extend(ops, make_std_options());
|
||||
$.extend(ops, getUrlVars());
|
||||
var str = $.map(ops,function (val, index) {
|
||||
return index + "=" + val;
|
||||
}).join("&");
|
||||
@ -180,9 +194,9 @@
|
||||
|
||||
function make_std_options() {
|
||||
var options = {};
|
||||
if (getPeriod() != 'havana') {
|
||||
options['period'] = getPeriod();
|
||||
}
|
||||
{# if (getRelease() != 'havana') {#}
|
||||
options['release'] = getRelease();
|
||||
{# }#}
|
||||
if (getMetric() != 'loc') {
|
||||
options['metric'] = getMetric();
|
||||
}
|
||||
@ -203,7 +217,7 @@
|
||||
reload();
|
||||
});
|
||||
|
||||
$(document).on('change', '#period', function (evt) {
|
||||
$(document).on('change', '#release', function (evt) {
|
||||
reload();
|
||||
});
|
||||
|
||||
@ -211,8 +225,8 @@
|
||||
reload();
|
||||
});
|
||||
|
||||
function getPeriod() {
|
||||
return $('#period').val()
|
||||
function getRelease() {
|
||||
return $('#release').val()
|
||||
}
|
||||
|
||||
function getMetric() {
|
||||
@ -225,6 +239,8 @@
|
||||
|
||||
</script>
|
||||
|
||||
{% block scripts %}{% endblock %}
|
||||
|
||||
{% endblock %}
|
||||
|
||||
{% block body %}
|
||||
@ -244,10 +260,9 @@
|
||||
<option value="loc">Lines of code</option>
|
||||
</select>
|
||||
</span>
|
||||
<span class="drop_period" style="float: right;">
|
||||
<label for="period">Period </label><select id="period" name="period">
|
||||
<span class="drop_release" style="float: right;">
|
||||
<label for="release">Release </label><select id="release" name="release">
|
||||
<option value="all">All times</option>
|
||||
<option value="six_months">Last 6 months</option>
|
||||
<option value="havana">Havana</option>
|
||||
<option value="grizzly">Grizzly</option>
|
||||
<option value="folsom">Folsom</option>
|
||||
@ -278,7 +293,3 @@
|
||||
|
||||
</div>
|
||||
{% endblock %}
|
||||
|
||||
{% macro link(base, title) -%}
|
||||
<a href="{{ base }}?metric={{ metric }}&period={{ period }}&project_type={{ project_type }}">{{ title }}</a>
|
||||
{%- endmacro %}
|
||||
|
||||
@ -4,16 +4,17 @@
|
||||
{{ module }}
|
||||
{% endblock %}
|
||||
|
||||
{% block scripts %}
|
||||
<script type="text/javascript">
|
||||
chartAndTableRenderer("/data/companies", "left_list", "left_chart", "/companies/", {company: "{{ company|encode }}" });
|
||||
timelineRenderer({company: "{{ company|encode }}" })
|
||||
</script>
|
||||
{% endblock %}
|
||||
|
||||
{% block left_frame %}
|
||||
|
||||
<h2>Contribution by companies</h2>
|
||||
|
||||
<script type="text/javascript">
|
||||
loadTable("left_list", "/data/companies", { module: "{{ module }}" });
|
||||
loadChart("left_chart", "/data/companies", { module: "{{ module }}", limit: 10 });
|
||||
loadTimeline({module: '{{ module }}'})
|
||||
</script>
|
||||
|
||||
<div id="left_chart" style="width: 100%; height: 350px;"></div>
|
||||
|
||||
<table id="left_list" class="display">
|
||||
@ -38,30 +39,25 @@
|
||||
{% for rec in commits %}
|
||||
<div style="padding-bottom: 1em;">
|
||||
<div style='float: left; '>
|
||||
<img src="{{ rec.email|gravatar(size=32) }}">
|
||||
<img src="{{ rec.author_email|gravatar(size=32) }}">
|
||||
</div>
|
||||
<div style="margin-left: 4em;">
|
||||
<div>
|
||||
{{ link('/engineers/' + rec.launchpad_id, rec.name) }}
|
||||
{# <a href="/engineers/{{ rec.launchpad_id }}?metric={{ metric }}&period={{ period }}&project_type={{ project_type }}">{{ rec.name }}</a>#}
|
||||
{% if rec.launchpad_id %}
|
||||
{{ rec.author|link('/engineers/' + rec.launchpad_id)|safe }}
|
||||
{% else %}
|
||||
{{ rec.author }}
|
||||
{% endif %}
|
||||
{% if rec.company %}
|
||||
(
|
||||
{{ link('/companies/' + rec.company, rec.company) }}
|
||||
{# <a href="/companies/{{ rec.company }}?metric={{ metric }}&period={{ period }}&project_type={{ project_type }}">{{ rec.company }}</a>#}
|
||||
{{ rec.company|link('/companies/' + rec.company)|safe }}
|
||||
)
|
||||
{% endif %}
|
||||
<em>{{ rec.date|datetimeformat }}</em>
|
||||
</div>
|
||||
|
||||
{% if rec.ref %}
|
||||
<div>{{ rec.ref|safe }}</div>
|
||||
{% endif %}
|
||||
<div>{{ rec.text }}</div>
|
||||
{% if rec.change_id %}
|
||||
<div>Change-Id: <a
|
||||
href="https://review.openstack.org/#q,{{ rec.change_id }},n,z">{{ rec.change_id }}</a>
|
||||
</div>
|
||||
{% endif %}
|
||||
<div><b>{{ rec.subject }}</b></div>
|
||||
<div style="white-space: pre-wrap;">{{ rec|commit_message|safe }}</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endfor %}
|
||||
|
||||
@ -4,16 +4,18 @@
|
||||
Overview
|
||||
{% endblock %}
|
||||
|
||||
{% block scripts %}
|
||||
<script type="text/javascript">
|
||||
chartAndTableRenderer("/data/companies", "left_list", "left_chart", "/companies/");
|
||||
chartAndTableRenderer("/data/modules", "right_list", "right_chart", "/modules/");
|
||||
timelineRenderer()
|
||||
</script>
|
||||
{% endblock %}
|
||||
|
||||
{% block left_frame %}
|
||||
|
||||
<h2>Contribution by companies</h2>
|
||||
|
||||
<script type="text/javascript">
|
||||
loadTable("left_list", "/data/companies");
|
||||
loadChart("left_chart", "/data/companies", {limit: 10});
|
||||
loadTimeline('')
|
||||
</script>
|
||||
|
||||
<div id="left_chart" style="width: 100%; height: 350px;"></div>
|
||||
|
||||
<table id="left_list" class="display">
|
||||
@ -35,11 +37,6 @@
|
||||
|
||||
<h2>Contribution by modules</h2>
|
||||
|
||||
<script type="text/javascript">
|
||||
loadTable("right_list", "/data/modules");
|
||||
loadChart("right_chart", "/data/modules", {limit: 10});
|
||||
</script>
|
||||
|
||||
<div id="right_chart" style="width: 100%; height: 350px;"></div>
|
||||
|
||||
<table id="right_list" class="display">
|
||||
452
dashboard/web.py
Normal file
452
dashboard/web.py
Normal file
@ -0,0 +1,452 @@
|
||||
import cgi
|
||||
import datetime
|
||||
import functools
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import urllib
|
||||
|
||||
import flask
|
||||
from flask.ext import gravatar as gravatar_ext
|
||||
import time
|
||||
|
||||
from dashboard import memory_storage
|
||||
from stackalytics.processor.persistent_storage import PersistentStorageFactory
|
||||
from stackalytics.processor.runtime_storage import RuntimeStorageFactory
|
||||
from stackalytics.processor import user_utils
|
||||
|
||||
DEBUG = True
|
||||
RUNTIME_STORAGE_URI = 'memcached://127.0.0.1:11211'
|
||||
PERSISTENT_STORAGE_URI = 'mongodb://localhost'
|
||||
|
||||
# create our little application :)
|
||||
app = flask.Flask(__name__)
|
||||
app.config.from_object(__name__)
|
||||
app.config.from_envvar('DASHBOARD_CONF', silent=True)
|
||||
|
||||
|
||||
def get_vault():
|
||||
vault = getattr(app, 'stackalytics_vault', None)
|
||||
if not vault:
|
||||
vault = {}
|
||||
vault['runtime_storage'] = RuntimeStorageFactory.get_storage(
|
||||
RUNTIME_STORAGE_URI)
|
||||
vault['persistent_storage'] = PersistentStorageFactory.get_storage(
|
||||
PERSISTENT_STORAGE_URI)
|
||||
vault['memory_storage'] = (
|
||||
memory_storage.MemoryStorageFactory.get_storage(
|
||||
memory_storage.MEMORY_STORAGE_CACHED,
|
||||
vault['runtime_storage'].get_update(os.getpid())))
|
||||
|
||||
releases = vault['persistent_storage'].get_releases()
|
||||
vault['releases'] = dict((r['release_name'].lower(), r)
|
||||
for r in releases)
|
||||
app.stackalytics_vault = vault
|
||||
return vault
|
||||
|
||||
|
||||
def get_memory_storage():
|
||||
return get_vault()['memory_storage']
|
||||
|
||||
|
||||
def record_filter(parameter_getter=lambda x: flask.request.args.get(x)):
|
||||
def decorator(f):
|
||||
@functools.wraps(f)
|
||||
def decorated_function(*args, **kwargs):
|
||||
|
||||
vault = get_vault()
|
||||
memory_storage = vault['memory_storage']
|
||||
record_ids = memory_storage.get_record_ids()
|
||||
|
||||
param = parameter_getter('modules')
|
||||
if param:
|
||||
record_ids &= memory_storage.get_record_ids_by_modules(
|
||||
param.split(','))
|
||||
|
||||
if 'launchpad_id' in kwargs:
|
||||
param = kwargs['launchpad_id']
|
||||
else:
|
||||
param = (parameter_getter('launchpad_id') or
|
||||
parameter_getter('launchpad_ids'))
|
||||
if param:
|
||||
record_ids &= memory_storage.get_record_ids_by_launchpad_ids(
|
||||
param.split(','))
|
||||
|
||||
if 'company' in kwargs:
|
||||
param = kwargs['company']
|
||||
else:
|
||||
param = (parameter_getter('company') or
|
||||
parameter_getter('companies'))
|
||||
if param:
|
||||
record_ids &= memory_storage.get_record_ids_by_companies(
|
||||
param.split(','))
|
||||
|
||||
param = parameter_getter('release') or parameter_getter('releases')
|
||||
if param:
|
||||
if param != 'all':
|
||||
record_ids &= memory_storage.get_record_ids_by_releases(
|
||||
c.lower() for c in param.split(','))
|
||||
|
||||
kwargs['records'] = memory_storage.get_records(record_ids)
|
||||
return f(*args, **kwargs)
|
||||
|
||||
return decorated_function
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
def aggregate_filter():
|
||||
def decorator(f):
|
||||
@functools.wraps(f)
|
||||
def decorated_function(*args, **kwargs):
|
||||
|
||||
metric_filter = lambda r: r['loc']
|
||||
metric_param = flask.request.args.get('metric')
|
||||
if metric_param:
|
||||
metric = metric_param.lower()
|
||||
if metric == 'commits':
|
||||
metric_filter = lambda r: 1
|
||||
elif metric != 'loc':
|
||||
raise Exception('Invalid metric %s' % metric)
|
||||
|
||||
kwargs['metric_filter'] = metric_filter
|
||||
return f(*args, **kwargs)
|
||||
|
||||
return decorated_function
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
def exception_handler():
|
||||
def decorator(f):
|
||||
@functools.wraps(f)
|
||||
def decorated_function(*args, **kwargs):
|
||||
try:
|
||||
return f(*args, **kwargs)
|
||||
except Exception as e:
|
||||
print e
|
||||
flask.abort(404)
|
||||
|
||||
return decorated_function
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
DEFAULT_METRIC = 'loc'
|
||||
DEFAULT_RELEASE = 'havana'
|
||||
DEFAULT_PROJECT_TYPE = 'incubation'
|
||||
|
||||
INDEPENDENT = '*independent'
|
||||
|
||||
METRIC_LABELS = {
|
||||
'loc': 'Lines of code',
|
||||
'commits': 'Commits',
|
||||
}
|
||||
|
||||
PROJECT_TYPES = {
|
||||
'core': ['core'],
|
||||
'incubation': ['core', 'incubation'],
|
||||
'all': ['core', 'incubation', 'dev'],
|
||||
}
|
||||
|
||||
ISSUE_TYPES = ['bug', 'blueprint']
|
||||
|
||||
DEFAULT_RECORDS_LIMIT = 10
|
||||
|
||||
|
||||
def templated(template=None):
|
||||
def decorator(f):
|
||||
@functools.wraps(f)
|
||||
def decorated_function(*args, **kwargs):
|
||||
|
||||
vault = get_vault()
|
||||
template_name = template
|
||||
if template_name is None:
|
||||
template_name = (flask.request.endpoint.replace('.', '/') +
|
||||
'.html')
|
||||
ctx = f(*args, **kwargs)
|
||||
if ctx is None:
|
||||
ctx = {}
|
||||
|
||||
# put parameters into template
|
||||
metric = flask.request.args.get('metric')
|
||||
if metric not in METRIC_LABELS:
|
||||
metric = None
|
||||
ctx['metric'] = metric or DEFAULT_METRIC
|
||||
ctx['metric_label'] = METRIC_LABELS[ctx['metric']]
|
||||
|
||||
release = flask.request.args.get('release')
|
||||
releases = vault['releases']
|
||||
if release:
|
||||
release = release.lower()
|
||||
if release not in releases:
|
||||
release = None
|
||||
else:
|
||||
release = releases[release]['release_name']
|
||||
ctx['release'] = (release or DEFAULT_RELEASE).lower()
|
||||
|
||||
return flask.render_template(template_name, **ctx)
|
||||
|
||||
return decorated_function
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
@app.route('/')
|
||||
@templated()
|
||||
def overview():
|
||||
pass
|
||||
|
||||
|
||||
def contribution_details(records, limit=DEFAULT_RECORDS_LIMIT):
|
||||
blueprints_map = {}
|
||||
bugs_map = {}
|
||||
commits = []
|
||||
loc = 0
|
||||
|
||||
for record in records:
|
||||
loc += record['loc']
|
||||
commits.append(record)
|
||||
blueprint = record['blueprint_id']
|
||||
if blueprint:
|
||||
if blueprint in blueprints_map:
|
||||
blueprints_map[blueprint].append(record)
|
||||
else:
|
||||
blueprints_map[blueprint] = [record]
|
||||
|
||||
bug = record['bug_id']
|
||||
if bug:
|
||||
if bug in bugs_map:
|
||||
bugs_map[bug].append(record)
|
||||
else:
|
||||
bugs_map[bug] = [record]
|
||||
|
||||
blueprints = sorted([{'id': key,
|
||||
'module': value[0]['module'],
|
||||
'records': value}
|
||||
for key, value in blueprints_map.iteritems()],
|
||||
key=lambda x: x['id'])
|
||||
bugs = sorted([{'id': key, 'records': value}
|
||||
for key, value in bugs_map.iteritems()],
|
||||
key=lambda x: x['id'])
|
||||
commits.sort(key=lambda x: x['date'], reverse=True)
|
||||
|
||||
result = {
|
||||
'blueprints': blueprints,
|
||||
'bugs': bugs,
|
||||
'commits': commits[0:limit],
|
||||
'loc': loc,
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
@app.route('/companies/<company>')
|
||||
@exception_handler()
|
||||
@templated()
|
||||
@record_filter()
|
||||
def company_details(company, records):
|
||||
details = contribution_details(records)
|
||||
details['company'] = company
|
||||
return details
|
||||
|
||||
|
||||
@app.route('/modules/<module>')
|
||||
@exception_handler()
|
||||
@templated()
|
||||
@record_filter()
|
||||
def module_details(module, records):
|
||||
details = contribution_details(records)
|
||||
details['module'] = module
|
||||
return details
|
||||
|
||||
|
||||
@app.route('/engineers/<launchpad_id>')
|
||||
@exception_handler()
|
||||
@templated()
|
||||
@record_filter()
|
||||
def engineer_details(launchpad_id, records):
|
||||
persistent_storage = get_vault()['persistent_storage']
|
||||
user = list(persistent_storage.get_users(launchpad_id=launchpad_id))[0]
|
||||
|
||||
details = contribution_details(records)
|
||||
details['launchpad_id'] = launchpad_id
|
||||
details['user'] = user
|
||||
return details
|
||||
|
||||
|
||||
@app.errorhandler(404)
|
||||
def page_not_found(e):
|
||||
return flask.render_template('404.html'), 404
|
||||
|
||||
|
||||
def _get_aggregated_stats(records, metric_filter, keys, param_id,
|
||||
param_title=None):
|
||||
param_title = param_title or param_id
|
||||
result = dict((c, 0) for c in keys)
|
||||
titles = {}
|
||||
for record in records:
|
||||
result[record[param_id]] += metric_filter(record)
|
||||
titles[record[param_id]] = record[param_title]
|
||||
|
||||
response = [{'id': r, 'metric': result[r], 'name': titles[r]}
|
||||
for r in result if result[r]]
|
||||
response.sort(key=lambda x: x['metric'], reverse=True)
|
||||
return response
|
||||
|
||||
|
||||
@app.route('/data/companies')
|
||||
@exception_handler()
|
||||
@record_filter()
|
||||
@aggregate_filter()
|
||||
def get_companies(records, metric_filter):
|
||||
response = _get_aggregated_stats(records, metric_filter,
|
||||
get_memory_storage().get_companies(),
|
||||
'company_name')
|
||||
return json.dumps(response)
|
||||
|
||||
|
||||
@app.route('/data/modules')
|
||||
@exception_handler()
|
||||
@record_filter()
|
||||
@aggregate_filter()
|
||||
def get_modules(records, metric_filter):
|
||||
response = _get_aggregated_stats(records, metric_filter,
|
||||
get_memory_storage().get_modules(),
|
||||
'module')
|
||||
return json.dumps(response)
|
||||
|
||||
|
||||
@app.route('/data/engineers')
|
||||
@exception_handler()
|
||||
@record_filter()
|
||||
@aggregate_filter()
|
||||
def get_engineers(records, metric_filter):
|
||||
response = _get_aggregated_stats(records, metric_filter,
|
||||
get_memory_storage().get_launchpad_ids(),
|
||||
'launchpad_id', 'author')
|
||||
return json.dumps(response)
|
||||
|
||||
|
||||
@app.route('/data/timeline')
|
||||
@exception_handler()
|
||||
@record_filter(parameter_getter=lambda x: flask.request.args.get(x)
|
||||
if (x != "release") and (x != "releases") else None)
|
||||
def timeline(records):
|
||||
|
||||
# find start and end dates
|
||||
release_name = flask.request.args.get('release')
|
||||
|
||||
if not release_name:
|
||||
release_name = DEFAULT_RELEASE
|
||||
else:
|
||||
release_name = release_name.lower()
|
||||
|
||||
releases = get_vault()['releases']
|
||||
if release_name not in releases:
|
||||
flask.abort(404)
|
||||
release = releases[release_name]
|
||||
|
||||
start_date = release_start_date = user_utils.timestamp_to_week(
|
||||
user_utils.date_to_timestamp(release['start_date']))
|
||||
end_date = release_end_date = user_utils.timestamp_to_week(
|
||||
user_utils.date_to_timestamp(release['end_date']))
|
||||
now = user_utils.timestamp_to_week(int(time.time()))
|
||||
|
||||
# expand start-end to year if needed
|
||||
if release_end_date - release_start_date < 52:
|
||||
expansion = (52 - (release_end_date - release_start_date)) // 2
|
||||
if release_end_date + expansion < now:
|
||||
end_date += expansion
|
||||
else:
|
||||
end_date = now
|
||||
start_date = end_date - 52
|
||||
|
||||
# empty stats for all weeks in range
|
||||
weeks = range(start_date, end_date)
|
||||
week_stat_loc = dict((c, 0) for c in weeks)
|
||||
week_stat_commits = dict((c, 0) for c in weeks)
|
||||
|
||||
# fill stats with the data
|
||||
for record in records:
|
||||
week = record['week']
|
||||
if week in weeks:
|
||||
week_stat_loc[week] += record['loc']
|
||||
week_stat_commits[week] += 1
|
||||
|
||||
# form arrays in format acceptable to timeline plugin
|
||||
array_loc = []
|
||||
array_commits = []
|
||||
array_commits_hl = []
|
||||
|
||||
for week in weeks:
|
||||
week_str = user_utils.week_to_date(week)
|
||||
array_loc.append([week_str, week_stat_loc[week]])
|
||||
if release_start_date <= week <= release_end_date:
|
||||
array_commits_hl.append([week_str, week_stat_commits[week]])
|
||||
array_commits.append([week_str, week_stat_commits[week]])
|
||||
|
||||
return json.dumps([array_commits, array_commits_hl, array_loc])
|
||||
|
||||
|
||||
# Jinja Filters
|
||||
|
||||
@app.template_filter('datetimeformat')
|
||||
def format_datetime(timestamp):
|
||||
return datetime.datetime.utcfromtimestamp(
|
||||
timestamp).strftime('%d %b %Y @ %H:%M')
|
||||
|
||||
|
||||
@app.template_filter('launchpadmodule')
|
||||
def format_launchpad_module_link(module):
|
||||
return '<a href="https://launchpad.net/%s">%s</a>' % (module, module)
|
||||
|
||||
|
||||
@app.template_filter('encode')
|
||||
def safe_encode(s):
|
||||
return urllib.quote_plus(s)
|
||||
|
||||
|
||||
@app.template_filter('link')
|
||||
def make_link(title, uri=None):
|
||||
return '<a href="%(uri)s">%(title)s</a>' % {'uri': uri, 'title': title}
|
||||
|
||||
|
||||
def clear_text(s):
|
||||
return cgi.escape(re.sub(r'\n{2,}', '\n', s, flags=re.MULTILINE))
|
||||
|
||||
|
||||
def link_blueprint(s, module):
|
||||
return re.sub(r'(blueprint\s+)([\w-]+)',
|
||||
r'\1<a href="https://blueprints.launchpad.net/' +
|
||||
module + r'/+spec/\2">\2</a>',
|
||||
s, flags=re.IGNORECASE)
|
||||
|
||||
|
||||
def link_bug(s):
|
||||
return re.sub(r'(bug\s+)#?([\d]{5,7})',
|
||||
r'\1<a href="https://bugs.launchpad.net/bugs/\2">\2</a>',
|
||||
s, flags=re.IGNORECASE)
|
||||
|
||||
|
||||
def link_change_id(s):
|
||||
return re.sub(r'\s+(I[0-9a-f]{40})',
|
||||
r' <a href="https://review.openstack.org/#q,\1,n,z">\1</a>',
|
||||
s)
|
||||
|
||||
|
||||
@app.template_filter('commit_message')
|
||||
def make_commit_message(record):
|
||||
|
||||
return link_change_id(link_bug(link_blueprint(clear_text(
|
||||
record['message']), record['module'])))
|
||||
|
||||
|
||||
gravatar = gravatar_ext.Gravatar(app,
|
||||
size=100,
|
||||
rating='g',
|
||||
default='wavatar',
|
||||
force_default=False,
|
||||
force_lower=False)
|
||||
|
||||
if __name__ == '__main__':
|
||||
app.run('0.0.0.0')
|
||||
13549
etc/default_data.json
Normal file
13549
etc/default_data.json
Normal file
File diff suppressed because it is too large
Load Diff
@ -1,94 +0,0 @@
|
||||
# domain employer [< yyyy-mm-dd]
|
||||
3ds.com Dassault Systèmes
|
||||
99cloud.net 99cloud
|
||||
alyseo.com Alyseo
|
||||
ansolabs.com Rackspace < 2012-07-20
|
||||
ansolabs.com Nebula
|
||||
atomia.com Atomia
|
||||
att.com AT&T
|
||||
attinteractive.com AT&T
|
||||
bigswitch.com Big Switch Networks
|
||||
b1-systems.de BL Systems
|
||||
brocade.com Brocade
|
||||
bull.net Bull
|
||||
canonical.com Canonical
|
||||
cern.ch CERN
|
||||
cisco.com Cisco Systems
|
||||
citrix.com Citrix
|
||||
cloud.com Citrix Systems
|
||||
cloudbau.de Cloudbau
|
||||
cloudscaling.com Cloudscaling
|
||||
dell.com Dell
|
||||
Dell.com Dell
|
||||
denali-systems.com Denali Systems
|
||||
dreamhost.com DreamHost
|
||||
emc.com EMC
|
||||
enovance.com eNovance
|
||||
fathomdb.com FathomDB
|
||||
gluster.com Red Hat
|
||||
griddynamics.com Grid Dynamics
|
||||
guardian.co.uk The Guardian
|
||||
hds.com Hitachi
|
||||
hp.com HP
|
||||
huawei.com Huawei
|
||||
ibm.com IBM
|
||||
inktank.com Inktank
|
||||
intel.com Intel
|
||||
internap.com Internap
|
||||
isi.edu University of Southern Carolina
|
||||
ispras.ru ISP RAS
|
||||
kt.com KT Corporation
|
||||
kth.se Kungliga Tekniska högskolan
|
||||
linux.vnet.ibm.com IBM
|
||||
locaweb.com.br Locaweb
|
||||
lahondaresearch.org La Honda Research
|
||||
managedit.ie Managed IT
|
||||
mellanox.com Mellanox
|
||||
memset.com Memset
|
||||
metacloud.com Metacloud
|
||||
midokura.com Midokura
|
||||
midokura.jp Midokura
|
||||
mirantis.com Mirantis
|
||||
mirantis.ru Mirantis
|
||||
nasa.gov NASA
|
||||
nebula.com Nebula
|
||||
nexenta.com Nexenta
|
||||
nec.co.jp NEC
|
||||
cq.jp.nec.com NEC
|
||||
da.jp.nec.com NEC
|
||||
mxw.nes.nec.co.jp NEC
|
||||
mxd.nes.nec.co.jp NEC
|
||||
netapp.com NetApp
|
||||
nicira.com Nicira
|
||||
nimbisservices.com Nimbis Services
|
||||
ntt.co.jp NTT
|
||||
ntt.com NTT
|
||||
nttdata.com NTT
|
||||
nttdata.co.jp NTT
|
||||
nttmcl.com NTT
|
||||
vertex.co.in NTT
|
||||
pednape StackOps
|
||||
pistoncloud.com Piston Cloud
|
||||
rackspace.co.uk Rackspace
|
||||
rackspace.com Rackspace
|
||||
radware.com Radware
|
||||
redhat.com Red Hat
|
||||
scality.com Scality
|
||||
sdsc.edu San Diego Supercomputer Center
|
||||
sina.com SINA
|
||||
software.dell.com Dell
|
||||
solidfire.com SolidFire
|
||||
suse.de SUSE
|
||||
suse.com SUSE
|
||||
suse.cz SUSE
|
||||
swiftstack.com SwiftStack
|
||||
thoughtworks.com ThoughtWorks
|
||||
umd.edu University of Maryland
|
||||
unimelb.edu.au University of Melbourne
|
||||
valinux.co.jp VA Linux
|
||||
vexxhost.com VexxHost
|
||||
virtualtech.jp Virtualtech
|
||||
vmware.com VMware
|
||||
wikimedia.org Wikimedia Foundation
|
||||
yahoo-inc.com Yahoo!
|
||||
zadarastorage.com Zadara Storage
|
||||
@ -1,257 +0,0 @@
|
||||
armamig@gmail.com Armando.Migliaccio@eu.citrix.com
|
||||
yogesh.srikrishnan@rackspace.com yogesh.srikrishnan@rackspace.com
|
||||
emagana@gmail.com eperdomo@cisco.com eperdomo@dhcp-171-71-119-164.cisco.com
|
||||
jeblair@hp.com jeblair@openstack.org
|
||||
rohitgarwalla@gmail.com roagarwa@cisco.com
|
||||
shweta.ap05@gmail.com shpadubi@cisco.com
|
||||
e0ne@e0ne.info ikolodyazhny@mirantis.com
|
||||
santhosh.m@thoughtworks.com santhom@thoughtworks.com
|
||||
john.garbutt@citrix.com john.garbutt@rackspace.com
|
||||
thingee@gmail.com mike.perez@dreamhost.com
|
||||
duncan.thomas@gmail.com duncan.thomas@hp.com
|
||||
adamg@canonical.com adam.gandelman@canonical.com
|
||||
admin@jakedahn.com jake@ansolabs.com
|
||||
amesserl@rackspace.com ant@openstack.org
|
||||
amigliaccio@internap.com Armando.Migliaccio@eu.citrix.com
|
||||
andrew@cloudscaling.com acs@parvuscaptus.com
|
||||
anne.gentle@rackspace.com anne@openstack.org
|
||||
anne@openstack.org anne@openstack.org
|
||||
armando.migliaccio@citrix.com Armando.Migliaccio@eu.citrix.com
|
||||
asomya@cisco.com asomya@cisco.com
|
||||
bcwaldon@gmail.com brian.waldon@rackspace.com
|
||||
bfschott@gmail.com bschott@isi.edu
|
||||
Bogott abogott@wikimedia.org
|
||||
brian.lamar@gmail.com brian.lamar@rackspace.com
|
||||
brian.lamar@rackspace.com brian.lamar@rackspace.com
|
||||
cbehrens@codestud.com cbehrens+github@codestud.com
|
||||
cbehrens+github@codestud.com cbehrens+github@codestud.com
|
||||
chiradeep@chiradeep-lt2 chiradeep@cloud.com
|
||||
chmouel@chmouel.com chmouel.boudjnah@rackspace.co.uk
|
||||
chmouel@enovance.com chmouel@chmouel.com
|
||||
chris.behrens@rackspace.com cbehrens@codestud.com
|
||||
chris@slicehost.com chris@pistoncloud.com
|
||||
chuck.short@canonical.com zulcss@ubuntu.com
|
||||
clayg@clayg-desktop clay.gerrard@gmail.com
|
||||
clay.gerrard@gmail.com clay.gerrard@gmail.com
|
||||
clay.gerrard@rackspace.com clay.gerrard@gmail.com
|
||||
code@term.ie code@term.ie
|
||||
corvus@gnu.org jeblair@hp.com
|
||||
corvus@inaugust.com jeblair@hp.com
|
||||
corywright@gmail.com cory.wright@rackspace.com
|
||||
cory.wright@rackspace.com corywright@gmail.com
|
||||
dan@nicira.com dan@nicira.com
|
||||
dan.prince@rackspace.com dprince@redhat.com
|
||||
danwent@dan-xs3-cs dan@nicira.com
|
||||
danwent@gmail.com dan@nicira.com
|
||||
Dave.Walker@Canonical.com dave.walker@canonical.com
|
||||
DaveWalker@ubuntu.com dave.walker@canonical.com
|
||||
DaveWalker@ubuntu.com Dave.Walker@canonical.com
|
||||
david.goetz@gmail.com david.goetz@rackspace.com
|
||||
david.hadas@gmail.com davidh@il.ibm.com
|
||||
devcamcar@illian.local devin.carlen@gmail.com
|
||||
devnull@brim.net gholt@rackspace.com
|
||||
Dietz matt.dietz@rackspace.com
|
||||
dolph.mathews@gmail.com dolph.mathews@rackspace.com
|
||||
doug.hellmann@gmail.com doug.hellmann@dreamhost.com
|
||||
dougw@sdsc.edu dweimer@gmail.com
|
||||
dpgoetz@gmail.com david.goetz@rackspace.com
|
||||
dt-github@xr7.org dtroyer@gmail.com
|
||||
Édouard edouard.thuleau@orange.com
|
||||
emellor@silver ewan.mellor@citrix.com
|
||||
enugaev@griddynamics.com reldan@oscloud.ru
|
||||
florian.hines@gmail.com syn@ronin.io
|
||||
gaurav@gluster.com gaurav@gluster.com
|
||||
ghe.rivero@gmail.com ghe@debian.org
|
||||
ghe.rivero@stackops.com ghe@debian.org
|
||||
gholt@brim.net gholt@rackspace.com
|
||||
gihub@highbridgellc.com github@highbridgellc.com
|
||||
github@anarkystic.com code@term.ie
|
||||
github@anarkystic.com github@anarkystic.com
|
||||
glange@rackspace.com greglange@gmail.com
|
||||
greglange+launchpad@gmail.com greglange@gmail.com
|
||||
heut2008@gmail.com yaguang.tang@canonical.com
|
||||
higginsd@gmail.com derekh@redhat.com
|
||||
ialekseev@griddynamics.com ilyaalekseyev@acm.org
|
||||
ilya@oscloud.ru ilyaalekseyev@acm.org
|
||||
itoumsn@shayol itoumsn@nttdata.co.jp
|
||||
jake@ansolabs.com jake@ansolabs.com
|
||||
jake@markupisart.com jake@ansolabs.com
|
||||
james.blair@rackspace.com jeblair@hp.com
|
||||
jaypipes@gmail.com jaypipes@gmail.com
|
||||
jesse@aire.local anotherjesse@gmail.com
|
||||
jesse@dancelamb anotherjesse@gmail.com
|
||||
jesse@gigantor.local anotherjesse@gmail.com
|
||||
jesse@ubuntu anotherjesse@gmail.com
|
||||
jian.wen@ubuntu.com jian.wen@canonical.com
|
||||
jkearney@nova.(none) josh@jk0.org
|
||||
jkearney@nova.(none) josh.kearney@pistoncloud.com
|
||||
jmckenty@joshua-mckentys-macbook-pro.local jmckenty@gmail.com
|
||||
jmckenty@yyj-dhcp171.corp.flock.com jmckenty@gmail.com
|
||||
joe@cloudscaling.com joe@swiftstack.com
|
||||
johannes@compute3.221.st johannes.erdfelt@rackspace.com
|
||||
johannes@erdfelt.com johannes.erdfelt@rackspace.com
|
||||
john.dickinson@rackspace.com me@not.mn
|
||||
john.eo@gmail.com john.eo@rackspace.com
|
||||
john.griffith8@gmail.com john.griffith@solidfire.com
|
||||
john.griffith@solidfire.com john.griffith@solidfire.com
|
||||
josh@jk0.org josh.kearney@pistoncloud.com
|
||||
josh.kearney@rackspace.com josh@jk0.org
|
||||
josh.kearney@rackspace.com josh.kearney@pistoncloud.com
|
||||
joshua.mckenty@nasa.gov jmckenty@gmail.com
|
||||
jpipes@serialcoder jaypipes@gmail.com
|
||||
jpipes@uberbox.gateway.2wire.net jaypipes@gmail.com
|
||||
jsuh@bespin jsuh@isi.edu
|
||||
jtran@attinteractive.com jhtran@att.com
|
||||
julien.danjou@enovance.com julien@danjou.info
|
||||
justin@fathomdb.com justin@fathomdb.com
|
||||
justinsb@justinsb-desktop justin@fathomdb.com
|
||||
kapil.foss@gmail.com kapil.foss@gmail.com
|
||||
ken.pepple@rabbityard.com ken.pepple@gmail.com
|
||||
ke.wu@nebula.com ke.wu@ibeca.me
|
||||
khaled.hussein@gmail.com khaled.hussein@rackspace.com
|
||||
Knouff philip.knouff@mailtrust.com
|
||||
Kölker jason@koelker.net
|
||||
kshileev@griddynamics.com kshileev@gmail.com
|
||||
laner@controller rlane@wikimedia.org
|
||||
letterj@racklabs.com letterj@gmail.com
|
||||
liem.m.nguyen@gmail.com liem_m_nguyen@hp.com
|
||||
liem.m.nguyen@hp.com liem_m_nguyen@hp.com
|
||||
Lopez aloga@ifca.unican.es
|
||||
lorin@isi.edu lorin@nimbisservices.com
|
||||
lrqrun@gmail.com lrqrun@gmail.com
|
||||
lzyeval@gmail.com zhongyue.nah@intel.com
|
||||
marcelo.martins@rackspace.com btorch@gmail.com
|
||||
masumotok@nttdata.co.jp masumotok@nttdata.co.jp
|
||||
masumoto masumotok@nttdata.co.jp
|
||||
matt.dietz@rackspace.com matt.dietz@rackspace.com
|
||||
matthew.dietz@gmail.com matt.dietz@rackspace.com
|
||||
matthewdietz@Matthew-Dietzs-MacBook-Pro.local matt.dietz@rackspace.com
|
||||
McConnell bmcconne@rackspace.com
|
||||
mdietz@openstack matt.dietz@rackspace.com
|
||||
mgius7096@gmail.com launchpad@markgius.com
|
||||
michael.barton@rackspace.com mike@weirdlooking.com
|
||||
michael.still@canonical.com mikal@stillhq.com
|
||||
mike-launchpad@weirdlooking.com mike@weirdlooking.com
|
||||
Moore joelbm24@gmail.com
|
||||
mordred@hudson mordred@inaugust.com
|
||||
nati.ueno@gmail.com ueno.nachi@lab.ntt.co.jp
|
||||
naveed.massjouni@rackspace.com naveedm9@gmail.com
|
||||
nelson@nelson-laptop russ@crynwr.com
|
||||
nirmal.ranganathan@rackspace.com rnirmal@gmail.com
|
||||
nirmal.ranganathan@rackspace.coom rnirmal@gmail.com
|
||||
nova@u4 ueno.nachi@lab.ntt.co.jp
|
||||
nsokolov@griddynamics.net nsokolov@griddynamics.com
|
||||
openstack@lab.ntt.co.jp ueno.nachi@lab.ntt.co.jp
|
||||
paul@openstack.org paul@openstack.org
|
||||
paul@substation9.com paul@openstack.org
|
||||
paul.voccio@rackspace.com paul@openstack.org
|
||||
pvoccio@castor.local paul@openstack.org
|
||||
ramana@venus.lekha.org rjuvvadi@hcl.com
|
||||
rclark@chat-blanc rick@openstack.org
|
||||
renuka.apte@citrix.com renuka.apte@citrix.com
|
||||
rick.harris@rackspace.com rconradharris@gmail.com
|
||||
rick@quasar.racklabs.com rconradharris@gmail.com
|
||||
root@bsirish.(none) sirish.bitra@gmail.com
|
||||
root@debian.ohthree.com amesserl@rackspace.com
|
||||
root@mirror.nasanebula.net vishvananda@gmail.com
|
||||
root@openstack2-api masumotok@nttdata.co.jp
|
||||
root@tonbuntu sleepsonthefloor@gmail.com
|
||||
root@ubuntu vishvananda@gmail.com
|
||||
rrjuvvadi@gmail.com rjuvvadi@hcl.com
|
||||
salv.orlando@gmail.com salvatore.orlando@eu.citrix.com
|
||||
sandy@sandywalsh.com sandy@darksecretsoftware.com
|
||||
sandy@sandywalsh.com sandy.walsh@rackspace.com
|
||||
sandy.walsh@rackspace.com sandy@darksecretsoftware.com
|
||||
sandy.walsh@rackspace.com sandy.walsh@rackspace.com
|
||||
sateesh.chodapuneedi@citrix.com sateesh.chodapuneedi@citrix.com
|
||||
SB justin@fathomdb.com
|
||||
sirish.bitra@gmail.com sirish.bitra@gmail.com
|
||||
sleepsonthefloor@gmail.com sleepsonthefloor@gmail.com
|
||||
Smith code@term.ie
|
||||
Sokolov nsokolov@griddynamics.com
|
||||
Somya asomya@cisco.com
|
||||
soren.hansen@rackspace.com soren@linux2go.dk
|
||||
soren@linux2go.dk soren.hansen@rackspace.com
|
||||
soren@openstack.org soren.hansen@rackspace.com
|
||||
sorhanse@cisco.com sorenhansen@rackspace.com
|
||||
sorlando@nicira.com salvatore.orlando@eu.citrix.com
|
||||
spam@andcheese.org sam@swiftstack.com
|
||||
superstack@superstack.org justin@fathomdb.com
|
||||
termie@preciousroy.local code@term.ie
|
||||
thuleau@gmail.com edouard1.thuleau@orange.com
|
||||
thuleau@gmail.com edouard.thuleau@orange.com
|
||||
tim.simpson4@gmail.com tim.simpson@rackspace.com
|
||||
todd@lapex todd@ansolabs.com
|
||||
todd@rubidine.com todd@ansolabs.com
|
||||
todd@rubidine.com xtoddx@gmail.com
|
||||
tpatil@vertex.co.in tushar.vitthal.patil@gmail.com
|
||||
Tran jtran@attinteractive.com
|
||||
treyemorris@gmail.com trey.morris@rackspace.com
|
||||
ttcl@mac.com troy.toman@rackspace.com
|
||||
Ueno ueno.nachi@lab.ntt.co.jp
|
||||
vishvananda@yahoo.com vishvananda@gmail.com
|
||||
vito.ordaz@gmail.com victor.rodionov@nexenta.com
|
||||
wenjianhn@gmail.com jian.wen@canonical.com
|
||||
will.wolf@rackspace.com throughnothing@gmail.com
|
||||
wwkeyboard@gmail.com aaron.lee@rackspace.com
|
||||
xchu@redhat.com xychu2008@gmail.com
|
||||
xtoddx@gmail.com todd@ansolabs.com
|
||||
xyj.asmy@gmail.com xyj.asmy@gmail.com
|
||||
yorik@ytaraday yorik.sar@gmail.com
|
||||
YS vivek.ys@gmail.com
|
||||
z-github@brim.net gholt@rackspace.com
|
||||
ziad.sawalha@rackspace.com github@highbridgellc.com
|
||||
z-launchpad@brim.net gholt@rackspace.com
|
||||
derek.morton25@gmail.com derek@networkwhisperer.com
|
||||
bartosz.gorski@ntti3.com bartosz.gorski@nttmcl.com
|
||||
launchpad@chmouel.com chmouel@chmouel.com
|
||||
launchpad@chmouel.com chmouel@enovance.com
|
||||
launchpad@chmouel.com chmouel@openstack.org
|
||||
imsplitbit@gmail.com dsalinas@rackspace.com
|
||||
clint@fewbar.com clint.byrum@hp.com
|
||||
clint@fewbar.com clint@ubuntu.com
|
||||
sbaker@redhat.com steve@stevebaker.org
|
||||
asalkeld@redhat.com angus@salkeld.id.au
|
||||
evgeniy@afonichev.com eafonichev@mirantis.com
|
||||
smoser@ubuntu.com scott.moser@canonical.com
|
||||
smoser@ubuntu.com smoser@brickies.net
|
||||
smoser@ubuntu.com smoser@canonical.com
|
||||
smoser@ubuntu.com ssmoser2@gmail.com
|
||||
jason@koelker.net jkoelker@rackspace.com
|
||||
john.garbutt@rackspace.com john@johngarbutt.com
|
||||
zhongyue.nah@intel.com lzyeval@gmail.com
|
||||
jiajun@unitedstack.com iamljj@gmail.com
|
||||
christophe.sauthier@ubuntu.com christophe.sauthier@gmail.com
|
||||
christophe.sauthier@ubuntu.com christophe@sauthier.com
|
||||
christophe.sauthier@objectif-libre.com christophe@sauthier.com
|
||||
aababilov@griddynamics.com ilovegnulinux@gmail.com
|
||||
yportnova@griddynamics.com yportnov@yahoo-inc.com
|
||||
mkislins@yahoo-inc.com mkislinska@griddynamics.com
|
||||
ryan.moore@hp.com rmoore08@gmail.com
|
||||
starodubcevna@gmail.com nstarodubtsev@mirantis.com
|
||||
lakhinder.walia@hds.com lakhindr@hotmail.com
|
||||
kanzhe@gmail.com kanzhe.jiang@bigswitch.com
|
||||
anita.kuno@enovance.com akuno@lavabit.com
|
||||
me@frostman.ru slukjanov@mirantis.com
|
||||
alexei.kornienko@gmail.com akornienko@mirantis.com
|
||||
nicolas@barcet.com nick.barcet@canonical.com
|
||||
nicolas@barcet.com nick@enovance.com
|
||||
nicolas@barcet.com nijaba@ubuntu.com
|
||||
graham.binns@canonical.com gmb@canonical.com
|
||||
graham.binns@canonical.com gmb@grahambinns.com
|
||||
graham.binns@canonical.com graham.binns@gmail.com
|
||||
graham.binns@canonical.com graham@canonical.com
|
||||
graham.binns@canonical.com graham@grahambinns.com
|
||||
emilien.macchi@stackops.com emilien.macchi@enovance.com
|
||||
emilien.macchi@stackops.com emilien@enovance.com
|
||||
swann.croiset@bull.net swann@oopss.org
|
||||
soulascedric@gmail.com cedric.soulas@cloudwatt.com
|
||||
simon.pasquier@bull.net pasquier.simon+launchpad@gmail.com
|
||||
simon.pasquier@bull.net pasquier.simon@gmail.com
|
||||
bogorodskiy@gmail.com novel@FreeBSD.org
|
||||
bogorodskiy@gmail.com rbogorodskiy@mirantis.com
|
||||
svilgelm@mirantis.com sergey.vilgelm@gmail.com
|
||||
robert.myers@rackspace.com robert_myers@earthlink.net
|
||||
raymond_pekowski@dell.com pekowski@gmail.com
|
||||
agorodnev@mirantis.com a.gorodnev@gmail.com
|
||||
rprikhodchenko@mirantis.com me@romcheg.me
|
||||
@ -1,17 +0,0 @@
|
||||
# user@domain employer
|
||||
anotherjesse@gmail.com Nebula
|
||||
bcwaldon@gmail.com Nebula
|
||||
code@term.ie Nebula
|
||||
dprince@redhat.com Red Hat
|
||||
github@anarkystic.com Nebula
|
||||
jake@ansolabs.com Nebula
|
||||
jaypipes@gmail.com AT&T
|
||||
jeblair@hp.com HP
|
||||
lzyeval@gmail.com Intel
|
||||
me@not.mn SwiftStack
|
||||
mordred@inaugust.com HP
|
||||
sleepsonthefloor@gmail.com Nebula
|
||||
soren@linux2go.dk Cisco
|
||||
vishvananda@gmail.com Nebula
|
||||
dtroyer@gmail.com Nebula
|
||||
|
||||
@ -1,36 +0,0 @@
|
||||
tatyana-leontovich Grid Dynamics
|
||||
vkhomenko Grid Dynamics
|
||||
cthiel-suse DE Telekom
|
||||
yorik-sar Mirantis
|
||||
gelbuhos Mirantis
|
||||
aababilov Grid Dynamics
|
||||
alexpilotti Cloudbase Solutions
|
||||
devananda HP
|
||||
heckj Nebula
|
||||
matt-sherborne Rackspace
|
||||
michael-ogorman Cisco Systems
|
||||
boris-42 Mirantis
|
||||
boris-42 *independent < 2013-04-10
|
||||
victor-r-howard Comcast
|
||||
amitry Comcast
|
||||
scollins Comcast
|
||||
w-emailme Comcast
|
||||
jasondunsmore Rackspace
|
||||
kannan Rightscale
|
||||
bob-melander Cisco Systems
|
||||
gabriel-hurley Nebula
|
||||
mathrock National Security Agency
|
||||
yosshy NEC
|
||||
johngarbutt Rackspace
|
||||
johngarbutt Citrix < 2013-02-01
|
||||
jean-baptiste-ransy Alyseo
|
||||
darren-birkett Rackspace
|
||||
lucasagomes Red Hat
|
||||
nobodycam HP
|
||||
cboylan HP
|
||||
dmllr SUSE
|
||||
therve HP
|
||||
hughsaunders Rackspace
|
||||
bruno-semperlotti Dassault Systèmes
|
||||
james-slagle Red Hat
|
||||
openstack *robots
|
||||
1128
etc/launchpad-ids
1128
etc/launchpad-ids
File diff suppressed because it is too large
Load Diff
@ -1,27 +0,0 @@
|
||||
[DEFAULT]
|
||||
# Run in debug mode?
|
||||
# debug = False
|
||||
|
||||
# Database parameters
|
||||
# db_driver = sqlite
|
||||
# db_user = operator
|
||||
# db_password = None
|
||||
# db_database = /opt/stack/data/stackalytics.sqlite
|
||||
# db_hostname = localhost
|
||||
|
||||
# Extensions
|
||||
# extensions = CommitsLOC,MessageDetails
|
||||
|
||||
# Root for all project sources. The tool will iterate over its contents
|
||||
# sources_root = /opt/stack/repos
|
||||
|
||||
# email mappings (e.g. collected from .mailmap files)
|
||||
# email_aliases = etc/email-aliases
|
||||
# mappings from domains to company names
|
||||
# domain2company = etc/domain-company
|
||||
# mappings from emails to company names
|
||||
# email2company = etc/email-company
|
||||
# mappings from launchpad ids to emails and user names
|
||||
# launchpad2email = etc/launchpad-ids
|
||||
# mappings from launchpad id to company name
|
||||
# launchpad2company = etc/launchpad-company
|
||||
24
etc/stackalytics.conf
Normal file
24
etc/stackalytics.conf
Normal file
@ -0,0 +1,24 @@
|
||||
[DEFAULT]
|
||||
# Run in debug mode?
|
||||
# debug = False
|
||||
|
||||
# Default data
|
||||
# default-data = etc/default_data.json
|
||||
|
||||
# The folder that holds all project sources to analyze
|
||||
# sources_root = ../metric-root-tmp
|
||||
|
||||
# Runtime storage URI
|
||||
# runtime_storage_uri = memcached://127.0.0.1:11211
|
||||
|
||||
# URI of persistent storage
|
||||
# persistent_storage_uri = mongodb://localhost
|
||||
|
||||
# Update persistent storage with default data
|
||||
# read-default-data = False
|
||||
|
||||
# Repo poll period in seconds
|
||||
# repo_poll_period = 300
|
||||
|
||||
# Address of update handler
|
||||
# frontend_update_address = http://user:user@localhost/update/%s
|
||||
121
etc/test_default_data.json
Normal file
121
etc/test_default_data.json
Normal file
@ -0,0 +1,121 @@
|
||||
{
|
||||
"users": [
|
||||
{
|
||||
"launchpad_id": "foo",
|
||||
"user_name": "Pupkin",
|
||||
"emails": ["a@a"],
|
||||
"companies": [
|
||||
{
|
||||
"company_name": "Uno",
|
||||
"end_date": "2013-Jan-01"
|
||||
},
|
||||
{
|
||||
"company_name": "Duo",
|
||||
"end_date": null
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
"companies": [
|
||||
{
|
||||
"company_name": "Mirantis",
|
||||
"domains": ["mirantis.com"]
|
||||
},
|
||||
{
|
||||
"company_name": "*independent",
|
||||
"domains": [""]
|
||||
},
|
||||
{
|
||||
"company_name": "Hewlett-Packard",
|
||||
"domains": ["hp.com"]
|
||||
},
|
||||
{
|
||||
"company_name": "Intel",
|
||||
"domains": ["intel.com"]
|
||||
}
|
||||
],
|
||||
|
||||
"repos": [
|
||||
{
|
||||
"branches": ["master"],
|
||||
"name": "Quantum Client",
|
||||
"type": "core",
|
||||
"uri": "git://github.com/openstack/python-quantumclient.git",
|
||||
"releases": [
|
||||
{
|
||||
"release_name": "Folsom",
|
||||
"tag_from": "folsom-1",
|
||||
"tag_to": "2.1"
|
||||
},
|
||||
{
|
||||
"release_name": "Grizzly",
|
||||
"tag_from": "2.1",
|
||||
"tag_to": "2.2.1"
|
||||
},
|
||||
{
|
||||
"release_name": "Havana",
|
||||
"tag_from": "2.2.1",
|
||||
"tag_to": "HEAD"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"branches": ["master"],
|
||||
"name": "Keystone",
|
||||
"type": "core",
|
||||
"uri": "git://github.com/openstack/keystone.git",
|
||||
"releases": [
|
||||
{
|
||||
"release_name": "Essex",
|
||||
"tag_from": "2011.3",
|
||||
"tag_to": "2012.1"
|
||||
},
|
||||
{
|
||||
"release_name": "Folsom",
|
||||
"tag_from": "2012.1",
|
||||
"tag_to": "2012.2"
|
||||
},
|
||||
{
|
||||
"release_name": "Grizzly",
|
||||
"tag_from": "2012.2",
|
||||
"tag_to": "2013.1"
|
||||
},
|
||||
{
|
||||
"release_name": "Havana",
|
||||
"tag_from": "2013.1",
|
||||
"tag_to": "HEAD"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
"releases": [
|
||||
{
|
||||
"release_name": "ALL",
|
||||
"start_date": "2010-May-01",
|
||||
"end_date": "now"
|
||||
},
|
||||
{
|
||||
"release_name": "Essex",
|
||||
"start_date": "2011-Oct-01",
|
||||
"end_date": "2012-Apr-01"
|
||||
},
|
||||
{
|
||||
"release_name": "Folsom",
|
||||
"start_date": "2012-Apr-01",
|
||||
"end_date": "2012-Oct-01"
|
||||
},
|
||||
{
|
||||
"release_name": "Grizzly",
|
||||
"start_date": "2012-Oct-01",
|
||||
"end_date": "2013-Apr-01"
|
||||
},
|
||||
{
|
||||
"release_name": "Havana",
|
||||
"start_date": "2013-Apr-01",
|
||||
"end_date": "now"
|
||||
}
|
||||
]
|
||||
|
||||
}
|
||||
@ -1,11 +1,12 @@
|
||||
d2to1>=0.2.10,<0.3
|
||||
pbr>=0.5.16,<0.6
|
||||
#MySQL-python
|
||||
#pysqlite
|
||||
#git+git://github.com/MetricsGrimoire/RepositoryHandler.git#egg=repositoryhandler-0.5
|
||||
#git+git://github.com/SoftwareIntrospectionLab/guilty.git#egg=guilty-2.1
|
||||
launchpadlib
|
||||
Flask>=0.9
|
||||
Flask-Gravatar
|
||||
oslo.config
|
||||
pylibmc
|
||||
iso8601
|
||||
launchpadlib
|
||||
http://tarballs.openstack.org/oslo.config/oslo.config-1.2.0a2.tar.gz#egg=oslo.config-1.2.0a2
|
||||
pbr>=0.5.16,<0.6
|
||||
psutil
|
||||
python-memcached
|
||||
pymongo
|
||||
sh
|
||||
six
|
||||
|
||||
1
stackalytics/__init__.py
Normal file
1
stackalytics/__init__.py
Normal file
@ -0,0 +1 @@
|
||||
__author__ = 'ishakhat'
|
||||
1
stackalytics/openstack/__init__.py
Normal file
1
stackalytics/openstack/__init__.py
Normal file
@ -0,0 +1 @@
|
||||
__author__ = 'ishakhat'
|
||||
0
stackalytics/openstack/common/__init__.py
Normal file
0
stackalytics/openstack/common/__init__.py
Normal file
68
stackalytics/openstack/common/importutils.py
Normal file
68
stackalytics/openstack/common/importutils.py
Normal file
@ -0,0 +1,68 @@
|
||||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2011 OpenStack Foundation.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Import related utilities and helper functions.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
|
||||
def import_class(import_str):
|
||||
"""Returns a class from a string including module and class."""
|
||||
mod_str, _sep, class_str = import_str.rpartition('.')
|
||||
try:
|
||||
__import__(mod_str)
|
||||
return getattr(sys.modules[mod_str], class_str)
|
||||
except (ValueError, AttributeError):
|
||||
raise ImportError('Class %s cannot be found (%s)' %
|
||||
(class_str,
|
||||
traceback.format_exception(*sys.exc_info())))
|
||||
|
||||
|
||||
def import_object(import_str, *args, **kwargs):
|
||||
"""Import a class and return an instance of it."""
|
||||
return import_class(import_str)(*args, **kwargs)
|
||||
|
||||
|
||||
def import_object_ns(name_space, import_str, *args, **kwargs):
|
||||
"""Tries to import object from default namespace.
|
||||
|
||||
Imports a class and return an instance of it, first by trying
|
||||
to find the class in a default namespace, then failing back to
|
||||
a full path if not found in the default namespace.
|
||||
"""
|
||||
import_value = "%s.%s" % (name_space, import_str)
|
||||
try:
|
||||
return import_class(import_value)(*args, **kwargs)
|
||||
except ImportError:
|
||||
return import_class(import_str)(*args, **kwargs)
|
||||
|
||||
|
||||
def import_module(import_str):
|
||||
"""Import a module."""
|
||||
__import__(import_str)
|
||||
return sys.modules[import_str]
|
||||
|
||||
|
||||
def try_import(import_str, default=None):
|
||||
"""Try to import a module and if it fails return default."""
|
||||
try:
|
||||
return import_module(import_str)
|
||||
except ImportError:
|
||||
return default
|
||||
169
stackalytics/openstack/common/jsonutils.py
Normal file
169
stackalytics/openstack/common/jsonutils.py
Normal file
@ -0,0 +1,169 @@
|
||||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2010 United States Government as represented by the
|
||||
# Administrator of the National Aeronautics and Space Administration.
|
||||
# Copyright 2011 Justin Santa Barbara
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
'''
|
||||
JSON related utilities.
|
||||
|
||||
This module provides a few things:
|
||||
|
||||
1) A handy function for getting an object down to something that can be
|
||||
JSON serialized. See to_primitive().
|
||||
|
||||
2) Wrappers around loads() and dumps(). The dumps() wrapper will
|
||||
automatically use to_primitive() for you if needed.
|
||||
|
||||
3) This sets up anyjson to use the loads() and dumps() wrappers if anyjson
|
||||
is available.
|
||||
'''
|
||||
|
||||
|
||||
import datetime
|
||||
import functools
|
||||
import inspect
|
||||
import itertools
|
||||
import json
|
||||
import types
|
||||
import xmlrpclib
|
||||
|
||||
import six
|
||||
|
||||
from stackalytics.openstack.common import timeutils
|
||||
|
||||
|
||||
_nasty_type_tests = [inspect.ismodule, inspect.isclass, inspect.ismethod,
|
||||
inspect.isfunction, inspect.isgeneratorfunction,
|
||||
inspect.isgenerator, inspect.istraceback, inspect.isframe,
|
||||
inspect.iscode, inspect.isbuiltin, inspect.isroutine,
|
||||
inspect.isabstract]
|
||||
|
||||
_simple_types = (types.NoneType, int, basestring, bool, float, long)
|
||||
|
||||
|
||||
def to_primitive(value, convert_instances=False, convert_datetime=True,
|
||||
level=0, max_depth=3):
|
||||
"""Convert a complex object into primitives.
|
||||
|
||||
Handy for JSON serialization. We can optionally handle instances,
|
||||
but since this is a recursive function, we could have cyclical
|
||||
data structures.
|
||||
|
||||
To handle cyclical data structures we could track the actual objects
|
||||
visited in a set, but not all objects are hashable. Instead we just
|
||||
track the depth of the object inspections and don't go too deep.
|
||||
|
||||
Therefore, convert_instances=True is lossy ... be aware.
|
||||
|
||||
"""
|
||||
# handle obvious types first - order of basic types determined by running
|
||||
# full tests on nova project, resulting in the following counts:
|
||||
# 572754 <type 'NoneType'>
|
||||
# 460353 <type 'int'>
|
||||
# 379632 <type 'unicode'>
|
||||
# 274610 <type 'str'>
|
||||
# 199918 <type 'dict'>
|
||||
# 114200 <type 'datetime.datetime'>
|
||||
# 51817 <type 'bool'>
|
||||
# 26164 <type 'list'>
|
||||
# 6491 <type 'float'>
|
||||
# 283 <type 'tuple'>
|
||||
# 19 <type 'long'>
|
||||
if isinstance(value, _simple_types):
|
||||
return value
|
||||
|
||||
if isinstance(value, datetime.datetime):
|
||||
if convert_datetime:
|
||||
return timeutils.strtime(value)
|
||||
else:
|
||||
return value
|
||||
|
||||
# value of itertools.count doesn't get caught by nasty_type_tests
|
||||
# and results in infinite loop when list(value) is called.
|
||||
if type(value) == itertools.count:
|
||||
return six.text_type(value)
|
||||
|
||||
# FIXME(vish): Workaround for LP bug 852095. Without this workaround,
|
||||
# tests that raise an exception in a mocked method that
|
||||
# has a @wrap_exception with a notifier will fail. If
|
||||
# we up the dependency to 0.5.4 (when it is released) we
|
||||
# can remove this workaround.
|
||||
if getattr(value, '__module__', None) == 'mox':
|
||||
return 'mock'
|
||||
|
||||
if level > max_depth:
|
||||
return '?'
|
||||
|
||||
# The try block may not be necessary after the class check above,
|
||||
# but just in case ...
|
||||
try:
|
||||
recursive = functools.partial(to_primitive,
|
||||
convert_instances=convert_instances,
|
||||
convert_datetime=convert_datetime,
|
||||
level=level,
|
||||
max_depth=max_depth)
|
||||
if isinstance(value, dict):
|
||||
return dict((k, recursive(v)) for k, v in value.iteritems())
|
||||
elif isinstance(value, (list, tuple)):
|
||||
return [recursive(lv) for lv in value]
|
||||
|
||||
# It's not clear why xmlrpclib created their own DateTime type, but
|
||||
# for our purposes, make it a datetime type which is explicitly
|
||||
# handled
|
||||
if isinstance(value, xmlrpclib.DateTime):
|
||||
value = datetime.datetime(*tuple(value.timetuple())[:6])
|
||||
|
||||
if convert_datetime and isinstance(value, datetime.datetime):
|
||||
return timeutils.strtime(value)
|
||||
elif hasattr(value, 'iteritems'):
|
||||
return recursive(dict(value.iteritems()), level=level + 1)
|
||||
elif hasattr(value, '__iter__'):
|
||||
return recursive(list(value))
|
||||
elif convert_instances and hasattr(value, '__dict__'):
|
||||
# Likely an instance of something. Watch for cycles.
|
||||
# Ignore class member vars.
|
||||
return recursive(value.__dict__, level=level + 1)
|
||||
else:
|
||||
if any(test(value) for test in _nasty_type_tests):
|
||||
return six.text_type(value)
|
||||
return value
|
||||
except TypeError:
|
||||
# Class objects are tricky since they may define something like
|
||||
# __iter__ defined but it isn't callable as list().
|
||||
return six.text_type(value)
|
||||
|
||||
|
||||
def dumps(value, default=to_primitive, **kwargs):
|
||||
return json.dumps(value, default=default, **kwargs)
|
||||
|
||||
|
||||
def loads(s):
|
||||
return json.loads(s)
|
||||
|
||||
|
||||
def load(s):
|
||||
return json.load(s)
|
||||
|
||||
|
||||
try:
|
||||
import anyjson
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
anyjson._modules.append((__name__, 'dumps', TypeError,
|
||||
'loads', ValueError, 'load'))
|
||||
anyjson.force_implementation(__name__)
|
||||
559
stackalytics/openstack/common/log.py
Normal file
559
stackalytics/openstack/common/log.py
Normal file
@ -0,0 +1,559 @@
|
||||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2011 OpenStack Foundation.
|
||||
# Copyright 2010 United States Government as represented by the
|
||||
# Administrator of the National Aeronautics and Space Administration.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Openstack logging handler.
|
||||
|
||||
This module adds to logging functionality by adding the option to specify
|
||||
a context object when calling the various log methods. If the context object
|
||||
is not specified, default formatting is used. Additionally, an instance uuid
|
||||
may be passed as part of the log message, which is intended to make it easier
|
||||
for admins to find messages related to a specific instance.
|
||||
|
||||
It also allows setting of formatting information through conf.
|
||||
|
||||
"""
|
||||
|
||||
import ConfigParser
|
||||
import cStringIO
|
||||
import inspect
|
||||
import itertools
|
||||
import logging
|
||||
import logging.config
|
||||
import logging.handlers
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
from oslo.config import cfg
|
||||
|
||||
# from quantum.openstack.common.gettextutils import _
|
||||
from stackalytics.openstack.common import importutils
|
||||
from stackalytics.openstack.common import jsonutils
|
||||
# from quantum.openstack.common import local
|
||||
|
||||
|
||||
_DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
|
||||
|
||||
common_cli_opts = [
|
||||
cfg.BoolOpt('debug',
|
||||
short='d',
|
||||
default=False,
|
||||
help='Print debugging output (set logging level to '
|
||||
'DEBUG instead of default WARNING level).'),
|
||||
cfg.BoolOpt('verbose',
|
||||
short='v',
|
||||
default=False,
|
||||
help='Print more verbose output (set logging level to '
|
||||
'INFO instead of default WARNING level).'),
|
||||
]
|
||||
|
||||
logging_cli_opts = [
|
||||
cfg.StrOpt('log-config',
|
||||
metavar='PATH',
|
||||
help='If this option is specified, the logging configuration '
|
||||
'file specified is used and overrides any other logging '
|
||||
'options specified. Please see the Python logging module '
|
||||
'documentation for details on logging configuration '
|
||||
'files.'),
|
||||
cfg.StrOpt('log-format',
|
||||
default=None,
|
||||
metavar='FORMAT',
|
||||
help='A logging.Formatter log message format string which may '
|
||||
'use any of the available logging.LogRecord attributes. '
|
||||
'This option is deprecated. Please use '
|
||||
'logging_context_format_string and '
|
||||
'logging_default_format_string instead.'),
|
||||
cfg.StrOpt('log-date-format',
|
||||
default=_DEFAULT_LOG_DATE_FORMAT,
|
||||
metavar='DATE_FORMAT',
|
||||
help='Format string for %%(asctime)s in log records. '
|
||||
'Default: %(default)s'),
|
||||
cfg.StrOpt('log-file',
|
||||
metavar='PATH',
|
||||
deprecated_name='logfile',
|
||||
help='(Optional) Name of log file to output to. '
|
||||
'If no default is set, logging will go to stdout.'),
|
||||
cfg.StrOpt('log-dir',
|
||||
deprecated_name='logdir',
|
||||
help='(Optional) The base directory used for relative '
|
||||
'--log-file paths'),
|
||||
cfg.BoolOpt('use-syslog',
|
||||
default=False,
|
||||
help='Use syslog for logging.'),
|
||||
cfg.StrOpt('syslog-log-facility',
|
||||
default='LOG_USER',
|
||||
help='syslog facility to receive log lines')
|
||||
]
|
||||
|
||||
generic_log_opts = [
|
||||
cfg.BoolOpt('use_stderr',
|
||||
default=True,
|
||||
help='Log output to standard error')
|
||||
]
|
||||
|
||||
log_opts = [
|
||||
cfg.StrOpt('logging_context_format_string',
|
||||
default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
|
||||
'%(name)s [%(request_id)s %(user)s %(tenant)s] '
|
||||
'%(instance)s%(message)s',
|
||||
help='format string to use for log messages with context'),
|
||||
cfg.StrOpt('logging_default_format_string',
|
||||
default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
|
||||
'%(name)s [-] %(instance)s%(message)s',
|
||||
help='format string to use for log messages without context'),
|
||||
cfg.StrOpt('logging_debug_format_suffix',
|
||||
default='%(funcName)s %(pathname)s:%(lineno)d',
|
||||
help='data to append to log format when level is DEBUG'),
|
||||
cfg.StrOpt('logging_exception_prefix',
|
||||
default='%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s '
|
||||
'%(instance)s',
|
||||
help='prefix each line of exception output with this format'),
|
||||
cfg.ListOpt('default_log_levels',
|
||||
default=[
|
||||
'amqplib=WARN',
|
||||
'sqlalchemy=WARN',
|
||||
'boto=WARN',
|
||||
'suds=INFO',
|
||||
'keystone=INFO',
|
||||
'eventlet.wsgi.server=WARN'
|
||||
],
|
||||
help='list of logger=LEVEL pairs'),
|
||||
cfg.BoolOpt('publish_errors',
|
||||
default=False,
|
||||
help='publish error events'),
|
||||
cfg.BoolOpt('fatal_deprecations',
|
||||
default=False,
|
||||
help='make deprecations fatal'),
|
||||
|
||||
# NOTE(mikal): there are two options here because sometimes we are handed
|
||||
# a full instance (and could include more information), and other times we
|
||||
# are just handed a UUID for the instance.
|
||||
cfg.StrOpt('instance_format',
|
||||
default='[instance: %(uuid)s] ',
|
||||
help='If an instance is passed with the log message, format '
|
||||
'it like this'),
|
||||
cfg.StrOpt('instance_uuid_format',
|
||||
default='[instance: %(uuid)s] ',
|
||||
help='If an instance UUID is passed with the log message, '
|
||||
'format it like this'),
|
||||
]
|
||||
|
||||
CONF = cfg.CONF
|
||||
CONF.register_cli_opts(common_cli_opts)
|
||||
CONF.register_cli_opts(logging_cli_opts)
|
||||
CONF.register_opts(generic_log_opts)
|
||||
CONF.register_opts(log_opts)
|
||||
|
||||
# our new audit level
|
||||
# NOTE(jkoelker) Since we synthesized an audit level, make the logging
|
||||
# module aware of it so it acts like other levels.
|
||||
logging.AUDIT = logging.INFO + 1
|
||||
logging.addLevelName(logging.AUDIT, 'AUDIT')
|
||||
|
||||
|
||||
try:
|
||||
NullHandler = logging.NullHandler
|
||||
except AttributeError: # NOTE(jkoelker) NullHandler added in Python 2.7
|
||||
class NullHandler(logging.Handler):
|
||||
def handle(self, record):
|
||||
pass
|
||||
|
||||
def emit(self, record):
|
||||
pass
|
||||
|
||||
def createLock(self):
|
||||
self.lock = None
|
||||
|
||||
|
||||
def _dictify_context(context):
|
||||
if context is None:
|
||||
return None
|
||||
if not isinstance(context, dict) and getattr(context, 'to_dict', None):
|
||||
context = context.to_dict()
|
||||
return context
|
||||
|
||||
|
||||
def _get_binary_name():
|
||||
return os.path.basename(inspect.stack()[-1][1])
|
||||
|
||||
|
||||
def _get_log_file_path(binary=None):
|
||||
logfile = CONF.log_file
|
||||
logdir = CONF.log_dir
|
||||
|
||||
if logfile and not logdir:
|
||||
return logfile
|
||||
|
||||
if logfile and logdir:
|
||||
return os.path.join(logdir, logfile)
|
||||
|
||||
if logdir:
|
||||
binary = binary or _get_binary_name()
|
||||
return '%s.log' % (os.path.join(logdir, binary),)
|
||||
|
||||
|
||||
class BaseLoggerAdapter(logging.LoggerAdapter):
|
||||
|
||||
def audit(self, msg, *args, **kwargs):
|
||||
self.log(logging.AUDIT, msg, *args, **kwargs)
|
||||
|
||||
|
||||
class LazyAdapter(BaseLoggerAdapter):
|
||||
def __init__(self, name='unknown', version='unknown'):
|
||||
self._logger = None
|
||||
self.extra = {}
|
||||
self.name = name
|
||||
self.version = version
|
||||
|
||||
@property
|
||||
def logger(self):
|
||||
if not self._logger:
|
||||
self._logger = getLogger(self.name, self.version)
|
||||
return self._logger
|
||||
|
||||
|
||||
class ContextAdapter(BaseLoggerAdapter):
|
||||
warn = logging.LoggerAdapter.warning
|
||||
|
||||
def __init__(self, logger, project_name, version_string):
|
||||
self.logger = logger
|
||||
self.project = project_name
|
||||
self.version = version_string
|
||||
|
||||
@property
|
||||
def handlers(self):
|
||||
return self.logger.handlers
|
||||
|
||||
def deprecated(self, msg, *args, **kwargs):
|
||||
stdmsg = _("Deprecated: %s") % msg
|
||||
if CONF.fatal_deprecations:
|
||||
self.critical(stdmsg, *args, **kwargs)
|
||||
raise DeprecatedConfig(msg=stdmsg)
|
||||
else:
|
||||
self.warn(stdmsg, *args, **kwargs)
|
||||
|
||||
def process(self, msg, kwargs):
|
||||
if 'extra' not in kwargs:
|
||||
kwargs['extra'] = {}
|
||||
extra = kwargs['extra']
|
||||
|
||||
context = kwargs.pop('context', None)
|
||||
# if not context:
|
||||
# context = getattr(local.store, 'context', None)
|
||||
if context:
|
||||
extra.update(_dictify_context(context))
|
||||
|
||||
instance = kwargs.pop('instance', None)
|
||||
instance_extra = ''
|
||||
if instance:
|
||||
instance_extra = CONF.instance_format % instance
|
||||
else:
|
||||
instance_uuid = kwargs.pop('instance_uuid', None)
|
||||
if instance_uuid:
|
||||
instance_extra = (CONF.instance_uuid_format
|
||||
% {'uuid': instance_uuid})
|
||||
extra.update({'instance': instance_extra})
|
||||
|
||||
extra.update({"project": self.project})
|
||||
extra.update({"version": self.version})
|
||||
extra['extra'] = extra.copy()
|
||||
return msg, kwargs
|
||||
|
||||
|
||||
class JSONFormatter(logging.Formatter):
|
||||
def __init__(self, fmt=None, datefmt=None):
|
||||
# NOTE(jkoelker) we ignore the fmt argument, but its still there
|
||||
# since logging.config.fileConfig passes it.
|
||||
self.datefmt = datefmt
|
||||
|
||||
def formatException(self, ei, strip_newlines=True):
|
||||
lines = traceback.format_exception(*ei)
|
||||
if strip_newlines:
|
||||
lines = [itertools.ifilter(
|
||||
lambda x: x,
|
||||
line.rstrip().splitlines()) for line in lines]
|
||||
lines = list(itertools.chain(*lines))
|
||||
return lines
|
||||
|
||||
def format(self, record):
|
||||
message = {'message': record.getMessage(),
|
||||
'asctime': self.formatTime(record, self.datefmt),
|
||||
'name': record.name,
|
||||
'msg': record.msg,
|
||||
'args': record.args,
|
||||
'levelname': record.levelname,
|
||||
'levelno': record.levelno,
|
||||
'pathname': record.pathname,
|
||||
'filename': record.filename,
|
||||
'module': record.module,
|
||||
'lineno': record.lineno,
|
||||
'funcname': record.funcName,
|
||||
'created': record.created,
|
||||
'msecs': record.msecs,
|
||||
'relative_created': record.relativeCreated,
|
||||
'thread': record.thread,
|
||||
'thread_name': record.threadName,
|
||||
'process_name': record.processName,
|
||||
'process': record.process,
|
||||
'traceback': None}
|
||||
|
||||
if hasattr(record, 'extra'):
|
||||
message['extra'] = record.extra
|
||||
|
||||
if record.exc_info:
|
||||
message['traceback'] = self.formatException(record.exc_info)
|
||||
|
||||
return jsonutils.dumps(message)
|
||||
|
||||
|
||||
def _create_logging_excepthook(product_name):
|
||||
def logging_excepthook(type, value, tb):
|
||||
extra = {}
|
||||
if CONF.verbose:
|
||||
extra['exc_info'] = (type, value, tb)
|
||||
getLogger(product_name).critical(str(value), **extra)
|
||||
return logging_excepthook
|
||||
|
||||
|
||||
class LogConfigError(Exception):
|
||||
|
||||
message = ('Error loading logging config %(log_config)s: %(err_msg)s')
|
||||
|
||||
def __init__(self, log_config, err_msg):
|
||||
self.log_config = log_config
|
||||
self.err_msg = err_msg
|
||||
|
||||
def __str__(self):
|
||||
return self.message % dict(log_config=self.log_config,
|
||||
err_msg=self.err_msg)
|
||||
|
||||
|
||||
def _load_log_config(log_config):
|
||||
try:
|
||||
logging.config.fileConfig(log_config)
|
||||
except ConfigParser.Error as exc:
|
||||
raise LogConfigError(log_config, str(exc))
|
||||
|
||||
|
||||
def setup(product_name):
|
||||
"""Setup logging."""
|
||||
if CONF.log_config:
|
||||
_load_log_config(CONF.log_config)
|
||||
else:
|
||||
_setup_logging_from_conf()
|
||||
sys.excepthook = _create_logging_excepthook(product_name)
|
||||
|
||||
|
||||
def set_defaults(logging_context_format_string):
|
||||
cfg.set_defaults(log_opts,
|
||||
logging_context_format_string=
|
||||
logging_context_format_string)
|
||||
|
||||
|
||||
def _find_facility_from_conf():
|
||||
facility_names = logging.handlers.SysLogHandler.facility_names
|
||||
facility = getattr(logging.handlers.SysLogHandler,
|
||||
CONF.syslog_log_facility,
|
||||
None)
|
||||
|
||||
if facility is None and CONF.syslog_log_facility in facility_names:
|
||||
facility = facility_names.get(CONF.syslog_log_facility)
|
||||
|
||||
if facility is None:
|
||||
valid_facilities = facility_names.keys()
|
||||
consts = ['LOG_AUTH', 'LOG_AUTHPRIV', 'LOG_CRON', 'LOG_DAEMON',
|
||||
'LOG_FTP', 'LOG_KERN', 'LOG_LPR', 'LOG_MAIL', 'LOG_NEWS',
|
||||
'LOG_AUTH', 'LOG_SYSLOG', 'LOG_USER', 'LOG_UUCP',
|
||||
'LOG_LOCAL0', 'LOG_LOCAL1', 'LOG_LOCAL2', 'LOG_LOCAL3',
|
||||
'LOG_LOCAL4', 'LOG_LOCAL5', 'LOG_LOCAL6', 'LOG_LOCAL7']
|
||||
valid_facilities.extend(consts)
|
||||
raise TypeError(('syslog facility must be one of: %s') %
|
||||
', '.join("'%s'" % fac
|
||||
for fac in valid_facilities))
|
||||
|
||||
return facility
|
||||
|
||||
|
||||
def _setup_logging_from_conf():
|
||||
log_root = getLogger(None).logger
|
||||
for handler in log_root.handlers:
|
||||
log_root.removeHandler(handler)
|
||||
|
||||
if CONF.use_syslog:
|
||||
facility = _find_facility_from_conf()
|
||||
syslog = logging.handlers.SysLogHandler(address='/dev/log',
|
||||
facility=facility)
|
||||
log_root.addHandler(syslog)
|
||||
|
||||
logpath = _get_log_file_path()
|
||||
if logpath:
|
||||
filelog = logging.handlers.WatchedFileHandler(logpath)
|
||||
log_root.addHandler(filelog)
|
||||
|
||||
if CONF.use_stderr:
|
||||
streamlog = ColorHandler()
|
||||
log_root.addHandler(streamlog)
|
||||
|
||||
elif not CONF.log_file:
|
||||
# pass sys.stdout as a positional argument
|
||||
# python2.6 calls the argument strm, in 2.7 it's stream
|
||||
streamlog = logging.StreamHandler(sys.stdout)
|
||||
log_root.addHandler(streamlog)
|
||||
|
||||
if CONF.publish_errors:
|
||||
handler = importutils.import_object(
|
||||
"quantum.openstack.common.log_handler.PublishErrorsHandler",
|
||||
logging.ERROR)
|
||||
log_root.addHandler(handler)
|
||||
|
||||
datefmt = CONF.log_date_format
|
||||
for handler in log_root.handlers:
|
||||
# NOTE(alaski): CONF.log_format overrides everything currently. This
|
||||
# should be deprecated in favor of context aware formatting.
|
||||
if CONF.log_format:
|
||||
handler.setFormatter(logging.Formatter(fmt=CONF.log_format,
|
||||
datefmt=datefmt))
|
||||
log_root.info('Deprecated: log_format is now deprecated and will '
|
||||
'be removed in the next release')
|
||||
else:
|
||||
handler.setFormatter(ContextFormatter(datefmt=datefmt))
|
||||
|
||||
if CONF.debug:
|
||||
log_root.setLevel(logging.DEBUG)
|
||||
elif CONF.verbose:
|
||||
log_root.setLevel(logging.INFO)
|
||||
else:
|
||||
log_root.setLevel(logging.WARNING)
|
||||
|
||||
for pair in CONF.default_log_levels:
|
||||
mod, _sep, level_name = pair.partition('=')
|
||||
level = logging.getLevelName(level_name)
|
||||
logger = logging.getLogger(mod)
|
||||
logger.setLevel(level)
|
||||
|
||||
_loggers = {}
|
||||
|
||||
|
||||
def getLogger(name='unknown', version='unknown'):
|
||||
if name not in _loggers:
|
||||
_loggers[name] = ContextAdapter(logging.getLogger(name),
|
||||
name,
|
||||
version)
|
||||
return _loggers[name]
|
||||
|
||||
|
||||
def getLazyLogger(name='unknown', version='unknown'):
|
||||
"""Returns lazy logger.
|
||||
|
||||
Creates a pass-through logger that does not create the real logger
|
||||
until it is really needed and delegates all calls to the real logger
|
||||
once it is created.
|
||||
"""
|
||||
return LazyAdapter(name, version)
|
||||
|
||||
|
||||
class WritableLogger(object):
|
||||
"""A thin wrapper that responds to `write` and logs."""
|
||||
|
||||
def __init__(self, logger, level=logging.INFO):
|
||||
self.logger = logger
|
||||
self.level = level
|
||||
|
||||
def write(self, msg):
|
||||
self.logger.log(self.level, msg)
|
||||
|
||||
|
||||
class ContextFormatter(logging.Formatter):
|
||||
"""A context.RequestContext aware formatter configured through flags.
|
||||
|
||||
The flags used to set format strings are: logging_context_format_string
|
||||
and logging_default_format_string. You can also specify
|
||||
logging_debug_format_suffix to append extra formatting if the log level is
|
||||
debug.
|
||||
|
||||
For information about what variables are available for the formatter see:
|
||||
http://docs.python.org/library/logging.html#formatter
|
||||
|
||||
"""
|
||||
|
||||
def format(self, record):
|
||||
"""Uses contextstring if request_id is set, otherwise default."""
|
||||
# NOTE(sdague): default the fancier formating params
|
||||
# to an empty string so we don't throw an exception if
|
||||
# they get used
|
||||
for key in ('instance', 'color'):
|
||||
if key not in record.__dict__:
|
||||
record.__dict__[key] = ''
|
||||
|
||||
if record.__dict__.get('request_id', None):
|
||||
self._fmt = CONF.logging_context_format_string
|
||||
else:
|
||||
self._fmt = CONF.logging_default_format_string
|
||||
|
||||
if (record.levelno == logging.DEBUG and
|
||||
CONF.logging_debug_format_suffix):
|
||||
self._fmt += " " + CONF.logging_debug_format_suffix
|
||||
|
||||
# Cache this on the record, Logger will respect our formated copy
|
||||
if record.exc_info:
|
||||
record.exc_text = self.formatException(record.exc_info, record)
|
||||
return logging.Formatter.format(self, record)
|
||||
|
||||
def formatException(self, exc_info, record=None):
|
||||
"""Format exception output with CONF.logging_exception_prefix."""
|
||||
if not record:
|
||||
return logging.Formatter.formatException(self, exc_info)
|
||||
|
||||
stringbuffer = cStringIO.StringIO()
|
||||
traceback.print_exception(exc_info[0], exc_info[1], exc_info[2],
|
||||
None, stringbuffer)
|
||||
lines = stringbuffer.getvalue().split('\n')
|
||||
stringbuffer.close()
|
||||
|
||||
if CONF.logging_exception_prefix.find('%(asctime)') != -1:
|
||||
record.asctime = self.formatTime(record, self.datefmt)
|
||||
|
||||
formatted_lines = []
|
||||
for line in lines:
|
||||
pl = CONF.logging_exception_prefix % record.__dict__
|
||||
fl = '%s%s' % (pl, line)
|
||||
formatted_lines.append(fl)
|
||||
return '\n'.join(formatted_lines)
|
||||
|
||||
|
||||
class ColorHandler(logging.StreamHandler):
|
||||
LEVEL_COLORS = {
|
||||
logging.DEBUG: '\033[00;32m', # GREEN
|
||||
logging.INFO: '\033[00;36m', # CYAN
|
||||
logging.AUDIT: '\033[01;36m', # BOLD CYAN
|
||||
logging.WARN: '\033[01;33m', # BOLD YELLOW
|
||||
logging.ERROR: '\033[01;31m', # BOLD RED
|
||||
logging.CRITICAL: '\033[01;31m', # BOLD RED
|
||||
}
|
||||
|
||||
def format(self, record):
|
||||
record.color = self.LEVEL_COLORS[record.levelno]
|
||||
return logging.StreamHandler.format(self, record)
|
||||
|
||||
|
||||
class DeprecatedConfig(Exception):
|
||||
message = ("Fatal call to deprecated config: %(msg)s")
|
||||
|
||||
def __init__(self, msg):
|
||||
super(Exception, self).__init__(self.message % dict(msg=msg))
|
||||
187
stackalytics/openstack/common/timeutils.py
Normal file
187
stackalytics/openstack/common/timeutils.py
Normal file
@ -0,0 +1,187 @@
|
||||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2011 OpenStack Foundation.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Time related utilities and helper functions.
|
||||
"""
|
||||
|
||||
import calendar
|
||||
import datetime
|
||||
|
||||
import iso8601
|
||||
|
||||
|
||||
# ISO 8601 extended time format with microseconds
|
||||
_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f'
|
||||
_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
|
||||
PERFECT_TIME_FORMAT = _ISO8601_TIME_FORMAT_SUBSECOND
|
||||
|
||||
|
||||
def isotime(at=None, subsecond=False):
|
||||
"""Stringify time in ISO 8601 format."""
|
||||
if not at:
|
||||
at = utcnow()
|
||||
st = at.strftime(_ISO8601_TIME_FORMAT
|
||||
if not subsecond
|
||||
else _ISO8601_TIME_FORMAT_SUBSECOND)
|
||||
tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC'
|
||||
st += ('Z' if tz == 'UTC' else tz)
|
||||
return st
|
||||
|
||||
|
||||
def parse_isotime(timestr):
|
||||
"""Parse time from ISO 8601 format."""
|
||||
try:
|
||||
return iso8601.parse_date(timestr)
|
||||
except iso8601.ParseError as e:
|
||||
raise ValueError(e.message)
|
||||
except TypeError as e:
|
||||
raise ValueError(e.message)
|
||||
|
||||
|
||||
def strtime(at=None, fmt=PERFECT_TIME_FORMAT):
|
||||
"""Returns formatted utcnow."""
|
||||
if not at:
|
||||
at = utcnow()
|
||||
return at.strftime(fmt)
|
||||
|
||||
|
||||
def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT):
|
||||
"""Turn a formatted time back into a datetime."""
|
||||
return datetime.datetime.strptime(timestr, fmt)
|
||||
|
||||
|
||||
def normalize_time(timestamp):
|
||||
"""Normalize time in arbitrary timezone to UTC naive object."""
|
||||
offset = timestamp.utcoffset()
|
||||
if offset is None:
|
||||
return timestamp
|
||||
return timestamp.replace(tzinfo=None) - offset
|
||||
|
||||
|
||||
def is_older_than(before, seconds):
|
||||
"""Return True if before is older than seconds."""
|
||||
if isinstance(before, basestring):
|
||||
before = parse_strtime(before).replace(tzinfo=None)
|
||||
return utcnow() - before > datetime.timedelta(seconds=seconds)
|
||||
|
||||
|
||||
def is_newer_than(after, seconds):
|
||||
"""Return True if after is newer than seconds."""
|
||||
if isinstance(after, basestring):
|
||||
after = parse_strtime(after).replace(tzinfo=None)
|
||||
return after - utcnow() > datetime.timedelta(seconds=seconds)
|
||||
|
||||
|
||||
def utcnow_ts():
|
||||
"""Timestamp version of our utcnow function."""
|
||||
return calendar.timegm(utcnow().timetuple())
|
||||
|
||||
|
||||
def utcnow():
|
||||
"""Overridable version of utils.utcnow."""
|
||||
if utcnow.override_time:
|
||||
try:
|
||||
return utcnow.override_time.pop(0)
|
||||
except AttributeError:
|
||||
return utcnow.override_time
|
||||
return datetime.datetime.utcnow()
|
||||
|
||||
|
||||
def iso8601_from_timestamp(timestamp):
|
||||
"""Returns a iso8601 formated date from timestamp."""
|
||||
return isotime(datetime.datetime.utcfromtimestamp(timestamp))
|
||||
|
||||
|
||||
utcnow.override_time = None
|
||||
|
||||
|
||||
def set_time_override(override_time=datetime.datetime.utcnow()):
|
||||
"""Overrides utils.utcnow.
|
||||
|
||||
Make it return a constant time or a list thereof, one at a time.
|
||||
"""
|
||||
utcnow.override_time = override_time
|
||||
|
||||
|
||||
def advance_time_delta(timedelta):
|
||||
"""Advance overridden time using a datetime.timedelta."""
|
||||
assert(not utcnow.override_time is None)
|
||||
try:
|
||||
for dt in utcnow.override_time:
|
||||
dt += timedelta
|
||||
except TypeError:
|
||||
utcnow.override_time += timedelta
|
||||
|
||||
|
||||
def advance_time_seconds(seconds):
|
||||
"""Advance overridden time by seconds."""
|
||||
advance_time_delta(datetime.timedelta(0, seconds))
|
||||
|
||||
|
||||
def clear_time_override():
|
||||
"""Remove the overridden time."""
|
||||
utcnow.override_time = None
|
||||
|
||||
|
||||
def marshall_now(now=None):
|
||||
"""Make an rpc-safe datetime with microseconds.
|
||||
|
||||
Note: tzinfo is stripped, but not required for relative times.
|
||||
"""
|
||||
if not now:
|
||||
now = utcnow()
|
||||
return dict(day=now.day, month=now.month, year=now.year, hour=now.hour,
|
||||
minute=now.minute, second=now.second,
|
||||
microsecond=now.microsecond)
|
||||
|
||||
|
||||
def unmarshall_time(tyme):
|
||||
"""Unmarshall a datetime dict."""
|
||||
return datetime.datetime(day=tyme['day'],
|
||||
month=tyme['month'],
|
||||
year=tyme['year'],
|
||||
hour=tyme['hour'],
|
||||
minute=tyme['minute'],
|
||||
second=tyme['second'],
|
||||
microsecond=tyme['microsecond'])
|
||||
|
||||
|
||||
def delta_seconds(before, after):
|
||||
"""Return the difference between two timing objects.
|
||||
|
||||
Compute the difference in seconds between two date, time, or
|
||||
datetime objects (as a float, to microsecond resolution).
|
||||
"""
|
||||
delta = after - before
|
||||
try:
|
||||
return delta.total_seconds()
|
||||
except AttributeError:
|
||||
return ((delta.days * 24 * 3600) + delta.seconds +
|
||||
float(delta.microseconds) / (10 ** 6))
|
||||
|
||||
|
||||
def is_soon(dt, window):
|
||||
"""Determines if time is going to happen in the next window seconds.
|
||||
|
||||
:params dt: the time
|
||||
:params window: minimum seconds to remain to consider the time not soon
|
||||
|
||||
:return: True if expiration is within the given duration
|
||||
"""
|
||||
soon = (utcnow() + datetime.timedelta(seconds=window))
|
||||
return normalize_time(dt) <= soon
|
||||
1
stackalytics/processor/__init__.py
Normal file
1
stackalytics/processor/__init__.py
Normal file
@ -0,0 +1 @@
|
||||
__author__ = 'ishakhat'
|
||||
177
stackalytics/processor/commit_processor.py
Normal file
177
stackalytics/processor/commit_processor.py
Normal file
@ -0,0 +1,177 @@
|
||||
# Copyright (c) 2013 Mirantis Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
from launchpadlib import launchpad
|
||||
from oslo.config import cfg
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
COMMIT_PROCESSOR_DUMMY = 0
|
||||
COMMIT_PROCESSOR_CACHED = 1
|
||||
|
||||
|
||||
class CommitProcessor(object):
|
||||
def __init__(self, persistent_storage):
|
||||
self.persistent_storage = persistent_storage
|
||||
|
||||
def process(self, commit_iterator):
|
||||
pass
|
||||
|
||||
|
||||
class DummyProcessor(CommitProcessor):
|
||||
def __init__(self, persistent_storage):
|
||||
super(DummyProcessor, self).__init__(persistent_storage)
|
||||
|
||||
def process(self, commit_iterator):
|
||||
return commit_iterator
|
||||
|
||||
|
||||
class CachedProcessor(CommitProcessor):
|
||||
def __init__(self, persistent_storage):
|
||||
super(CachedProcessor, self).__init__(persistent_storage)
|
||||
|
||||
companies = persistent_storage.get_companies()
|
||||
self.domains_index = {}
|
||||
for company in companies:
|
||||
for domain in company['domains']:
|
||||
self.domains_index[domain] = company['company_name']
|
||||
|
||||
users = persistent_storage.get_users()
|
||||
self.users_index = {}
|
||||
for user in users:
|
||||
for email in user['emails']:
|
||||
self.users_index[email] = user
|
||||
|
||||
LOG.debug('Cached commit processor is instantiated')
|
||||
|
||||
def _find_company(self, companies, date):
|
||||
for r in companies:
|
||||
if date < r['end_date']:
|
||||
return r['company_name']
|
||||
return companies[-1]['company_name']
|
||||
|
||||
def _get_company_by_email(self, email):
|
||||
name, at, domain = email.partition('@')
|
||||
if domain:
|
||||
parts = domain.split('.')
|
||||
for i in range(len(parts), 1, -1):
|
||||
m = '.'.join(parts[len(parts) - i:])
|
||||
if m in self.domains_index:
|
||||
return self.domains_index[m]
|
||||
return None
|
||||
|
||||
def _unknown_user_email(self, email):
|
||||
|
||||
lp_profile = None
|
||||
if not re.match(r'[^@]+@[^@]+\.[^@]+', email):
|
||||
LOG.debug('User email is not valid %s' % email)
|
||||
else:
|
||||
LOG.debug('Lookup user email %s at Launchpad' % email)
|
||||
lp = launchpad.Launchpad.login_anonymously(cfg.CONF.launchpad_user)
|
||||
try:
|
||||
lp_profile = lp.people.getByEmail(email=email)
|
||||
except Exception as error:
|
||||
LOG.warn('Lookup of email %s failed %s' %
|
||||
(email, error.message))
|
||||
if not lp_profile:
|
||||
# user is not found in Launchpad, create dummy record for commit
|
||||
# update
|
||||
LOG.debug('Email is not found at Launchpad, mapping to nobody')
|
||||
user = {
|
||||
'launchpad_id': None,
|
||||
'companies': [{
|
||||
'company_name': self.domains_index[''],
|
||||
'end_date': 0
|
||||
}]
|
||||
}
|
||||
else:
|
||||
# get user's launchpad id from his profile
|
||||
launchpad_id = lp_profile.name
|
||||
LOG.debug('Found user %s' % launchpad_id)
|
||||
|
||||
# check if user with launchpad_id exists in persistent storage
|
||||
persistent_user_iterator = self.persistent_storage.get_users(
|
||||
launchpad_id=launchpad_id)
|
||||
|
||||
for persistent_user in persistent_user_iterator:
|
||||
break
|
||||
else:
|
||||
persistent_user = None
|
||||
|
||||
if persistent_user:
|
||||
# user already exist, merge
|
||||
LOG.debug('User exists in persistent storage, add new email')
|
||||
persistent_user_email = persistent_user['emails'][0]
|
||||
if persistent_user_email not in self.users_index:
|
||||
raise Exception('User index is not valid')
|
||||
user = self.users_index[persistent_user_email]
|
||||
user['emails'].append(email)
|
||||
self.persistent_storage.update_user(user)
|
||||
else:
|
||||
# add new user
|
||||
LOG.debug('Add new user into persistent storage')
|
||||
company = (self._get_company_by_email(email) or
|
||||
self.domains_index[''])
|
||||
user = {
|
||||
'launchpad_id': launchpad_id,
|
||||
'user_name': lp_profile.display_name,
|
||||
'emails': [email],
|
||||
'companies': [{
|
||||
'company_name': company,
|
||||
'end_date': 0,
|
||||
}],
|
||||
}
|
||||
self.persistent_storage.insert_user(user)
|
||||
|
||||
# update local index
|
||||
self.users_index[email] = user
|
||||
return user
|
||||
|
||||
def _update_commit_with_user_data(self, commit):
|
||||
email = commit['author_email'].lower()
|
||||
if email in self.users_index:
|
||||
user = self.users_index[email]
|
||||
else:
|
||||
user = self._unknown_user_email(email)
|
||||
commit['launchpad_id'] = user['launchpad_id']
|
||||
company = self._get_company_by_email(email)
|
||||
if not company:
|
||||
company = self._find_company(user['companies'], commit['date'])
|
||||
commit['company_name'] = company
|
||||
|
||||
def process(self, commit_iterator):
|
||||
|
||||
for commit in commit_iterator:
|
||||
self._update_commit_with_user_data(commit)
|
||||
|
||||
yield commit
|
||||
|
||||
|
||||
class CommitProcessorFactory(object):
|
||||
@staticmethod
|
||||
def get_processor(commit_processor_type, persistent_storage):
|
||||
LOG.debug('Factory is asked for commit processor type %s' %
|
||||
commit_processor_type)
|
||||
if commit_processor_type == COMMIT_PROCESSOR_DUMMY:
|
||||
return DummyProcessor(persistent_storage)
|
||||
elif commit_processor_type == COMMIT_PROCESSOR_CACHED:
|
||||
return CachedProcessor(persistent_storage)
|
||||
else:
|
||||
raise Exception('Unknown commit processor type %s' %
|
||||
commit_processor_type)
|
||||
164
stackalytics/processor/main.py
Normal file
164
stackalytics/processor/main.py
Normal file
@ -0,0 +1,164 @@
|
||||
# Copyright (c) 2013 Mirantis Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import time
|
||||
|
||||
from oslo.config import cfg
|
||||
import psutil
|
||||
from psutil import _error
|
||||
import sh
|
||||
|
||||
from stackalytics.openstack.common import log as logging
|
||||
from stackalytics.openstack.common.timeutils import iso8601_from_timestamp
|
||||
from stackalytics.processor import commit_processor
|
||||
from stackalytics.processor.persistent_storage import PersistentStorageFactory
|
||||
from stackalytics.processor.runtime_storage import RuntimeStorageFactory
|
||||
from stackalytics.processor.vcs import VcsFactory
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
OPTS = [
|
||||
cfg.StrOpt('default-data', default='etc/default_data.json',
|
||||
help='Default data'),
|
||||
cfg.StrOpt('sources-root', default=None, required=True,
|
||||
help='The folder that holds all project sources to analyze'),
|
||||
cfg.StrOpt('runtime-storage-uri', default='memcached://127.0.0.1:11211',
|
||||
help='Storage URI'),
|
||||
cfg.StrOpt('frontend-update-address',
|
||||
default='http://user:user@localhost/update/%s',
|
||||
help='Address of update handler'),
|
||||
cfg.StrOpt('repo-poll-period', default='300',
|
||||
help='Repo poll period in seconds'),
|
||||
cfg.StrOpt('persistent-storage-uri', default='mongodb://localhost',
|
||||
help='URI of persistent storage'),
|
||||
cfg.BoolOpt('sync-default-data', default=False,
|
||||
help='Update persistent storage with default data. '
|
||||
'Existing data is not overwritten'),
|
||||
cfg.BoolOpt('force-sync-default-data', default=False,
|
||||
help='Completely overwrite persistent storage with the '
|
||||
'default data'),
|
||||
cfg.StrOpt('launchpad-user', default='stackalytics-bot',
|
||||
help='User to access Launchpad'),
|
||||
]
|
||||
|
||||
|
||||
def get_pids():
|
||||
uwsgi_dict = {}
|
||||
for pid in psutil.get_pid_list():
|
||||
try:
|
||||
p = psutil.Process(pid)
|
||||
if p.cmdline and p.cmdline[0].find('/uwsgi '):
|
||||
uwsgi_dict[p.pid] = p.parent
|
||||
except _error.NoSuchProcess:
|
||||
# the process may disappear after get_pid_list call, ignore it
|
||||
pass
|
||||
|
||||
result = set()
|
||||
for pid in uwsgi_dict:
|
||||
if uwsgi_dict[pid] in uwsgi_dict:
|
||||
result.add(pid)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def update_pid(pid):
|
||||
url = cfg.CONF.frontend_update_address % pid
|
||||
sh.curl(url)
|
||||
|
||||
|
||||
def update_pids(runtime_storage):
|
||||
pids = get_pids()
|
||||
if not pids:
|
||||
return
|
||||
runtime_storage.active_pids(pids)
|
||||
current_time = time.time()
|
||||
for pid in pids:
|
||||
if current_time > runtime_storage.get_pid_update_time(pid):
|
||||
update_pid(pid)
|
||||
return current_time
|
||||
|
||||
|
||||
def process_repo(repo, runtime_storage, processor):
|
||||
uri = repo['uri']
|
||||
LOG.debug('Processing repo uri %s' % uri)
|
||||
|
||||
vcs = VcsFactory.get_vcs(repo)
|
||||
vcs.fetch()
|
||||
|
||||
for branch in repo['branches']:
|
||||
LOG.debug('Processing repo %s, branch %s' % (uri, branch))
|
||||
|
||||
head_commit_id = runtime_storage.get_head_commit_id(uri, branch)
|
||||
|
||||
commit_iterator = vcs.log(branch, head_commit_id)
|
||||
processed_commit_iterator = processor.process(commit_iterator)
|
||||
runtime_storage.set_records(processed_commit_iterator)
|
||||
|
||||
head_commit_id = vcs.get_head_commit_id(branch)
|
||||
runtime_storage.set_head_commit_id(uri, branch, head_commit_id)
|
||||
|
||||
|
||||
def update_repos(runtime_storage, persistent_storage):
|
||||
current_time = time.time()
|
||||
repo_update_time = runtime_storage.get_repo_update_time()
|
||||
|
||||
if current_time < repo_update_time:
|
||||
LOG.info('The next update is scheduled at %s. Skipping' %
|
||||
iso8601_from_timestamp(repo_update_time))
|
||||
return
|
||||
|
||||
repos = persistent_storage.get_repos()
|
||||
processor = commit_processor.CommitProcessorFactory.get_processor(
|
||||
commit_processor.COMMIT_PROCESSOR_CACHED,
|
||||
persistent_storage)
|
||||
|
||||
for repo in repos:
|
||||
process_repo(repo, runtime_storage, processor)
|
||||
|
||||
runtime_storage.set_repo_update_time(time.time() +
|
||||
int(cfg.CONF.repo_poll_period))
|
||||
|
||||
|
||||
def main():
|
||||
# init conf and logging
|
||||
conf = cfg.CONF
|
||||
conf.register_cli_opts(OPTS)
|
||||
conf.register_opts(OPTS)
|
||||
conf()
|
||||
|
||||
logging.setup('stackalytics')
|
||||
LOG.info('Logging enabled')
|
||||
|
||||
persistent_storage = PersistentStorageFactory.get_storage(
|
||||
cfg.CONF.persistent_storage_uri)
|
||||
|
||||
if conf.sync_default_data or conf.force_sync_default_data:
|
||||
LOG.info('Going to synchronize persistent storage with default data '
|
||||
'from file %s' % cfg.CONF.default_data)
|
||||
persistent_storage.sync(cfg.CONF.default_data,
|
||||
force=conf.force_sync_default_data)
|
||||
return 0
|
||||
|
||||
runtime_storage = RuntimeStorageFactory.get_storage(
|
||||
cfg.CONF.runtime_storage_uri)
|
||||
|
||||
update_pids(runtime_storage)
|
||||
|
||||
update_repos(runtime_storage, persistent_storage)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
150
stackalytics/processor/persistent_storage.py
Normal file
150
stackalytics/processor/persistent_storage.py
Normal file
@ -0,0 +1,150 @@
|
||||
# Copyright (c) 2013 Mirantis Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
|
||||
import pymongo
|
||||
|
||||
from stackalytics.processor.user_utils import normalize_user
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PersistentStorage(object):
|
||||
def __init__(self, uri):
|
||||
pass
|
||||
|
||||
def sync(self, default_data_file_name, force=False):
|
||||
if force:
|
||||
self.clean_all()
|
||||
|
||||
default_data = self._read_default_persistent_storage(
|
||||
default_data_file_name)
|
||||
|
||||
self._build_index(default_data['repos'], 'uri',
|
||||
self.get_repos, self.insert_repo)
|
||||
self._build_index(default_data['companies'], 'company_name',
|
||||
self.get_companies, self.insert_company)
|
||||
self._build_index(default_data['users'], 'launchpad_id',
|
||||
self.get_users, self.insert_user)
|
||||
self._build_index(default_data['releases'], 'release_name',
|
||||
self.get_releases, self.insert_release)
|
||||
|
||||
def _build_index(self, default_data, primary_key, getter, inserter):
|
||||
# loads all items from persistent storage
|
||||
existing_items = set([item[primary_key] for item in getter()])
|
||||
# inserts items from default storage that are not in persistent storage
|
||||
map(inserter, [item for item in default_data
|
||||
if item[primary_key] not in existing_items])
|
||||
|
||||
def get_companies(self, **criteria):
|
||||
pass
|
||||
|
||||
def insert_company(self, company):
|
||||
pass
|
||||
|
||||
def get_repos(self, **criteria):
|
||||
pass
|
||||
|
||||
def insert_repo(self, repo):
|
||||
pass
|
||||
|
||||
def get_users(self, **criteria):
|
||||
pass
|
||||
|
||||
def insert_user(self, user):
|
||||
pass
|
||||
|
||||
def update_user(self, user):
|
||||
pass
|
||||
|
||||
def get_releases(self, **criteria):
|
||||
pass
|
||||
|
||||
def insert_release(self, release):
|
||||
pass
|
||||
|
||||
def clean_all(self):
|
||||
pass
|
||||
|
||||
def _read_default_persistent_storage(self, file_name):
|
||||
try:
|
||||
with open(file_name, 'r') as content_file:
|
||||
content = content_file.read()
|
||||
return json.loads(content)
|
||||
except Exception as e:
|
||||
LOG.error('Error while reading config: %s' % e)
|
||||
|
||||
|
||||
class MongodbStorage(PersistentStorage):
|
||||
def __init__(self, uri):
|
||||
super(MongodbStorage, self).__init__(uri)
|
||||
|
||||
self.client = pymongo.MongoClient(uri)
|
||||
self.mongo = self.client.stackalytics
|
||||
|
||||
self.mongo.companies.create_index([("company", pymongo.ASCENDING)])
|
||||
self.mongo.repos.create_index([("uri", pymongo.ASCENDING)])
|
||||
self.mongo.users.create_index([("launchpad_id", pymongo.ASCENDING)])
|
||||
self.mongo.releases.create_index([("releases", pymongo.ASCENDING)])
|
||||
|
||||
LOG.debug('Mongodb storage is created')
|
||||
|
||||
def clean_all(self):
|
||||
LOG.debug('Clear all tables')
|
||||
self.mongo.companies.remove()
|
||||
self.mongo.repos.remove()
|
||||
self.mongo.users.remove()
|
||||
self.mongo.releases.remove()
|
||||
|
||||
def get_companies(self, **criteria):
|
||||
return self.mongo.companies.find(criteria)
|
||||
|
||||
def insert_company(self, company):
|
||||
self.mongo.companies.insert(company)
|
||||
|
||||
def get_repos(self, **criteria):
|
||||
return self.mongo.repos.find(criteria)
|
||||
|
||||
def insert_repo(self, repo):
|
||||
self.mongo.repos.insert(repo)
|
||||
|
||||
def get_users(self, **criteria):
|
||||
return self.mongo.users.find(criteria)
|
||||
|
||||
def insert_user(self, user):
|
||||
self.mongo.users.insert(normalize_user(user))
|
||||
|
||||
def update_user(self, user):
|
||||
normalize_user(user)
|
||||
launchpad_id = user['launchpad_id']
|
||||
self.mongo.users.update({'launchpad_id': launchpad_id}, user)
|
||||
|
||||
def get_releases(self, **criteria):
|
||||
return self.mongo.releases.find(criteria)
|
||||
|
||||
def insert_release(self, release):
|
||||
self.mongo.releases.insert(release)
|
||||
|
||||
|
||||
class PersistentStorageFactory(object):
|
||||
@staticmethod
|
||||
def get_storage(uri):
|
||||
LOG.debug('Persistent storage is requested for uri %s' % uri)
|
||||
match = re.search(r'^mongodb:\/\/', uri)
|
||||
if match:
|
||||
return MongodbStorage(uri)
|
||||
196
stackalytics/processor/runtime_storage.py
Normal file
196
stackalytics/processor/runtime_storage.py
Normal file
@ -0,0 +1,196 @@
|
||||
# Copyright (c) 2013 Mirantis Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
|
||||
import re
|
||||
import urllib
|
||||
|
||||
import memcache
|
||||
from oslo.config import cfg
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RuntimeStorage(object):
|
||||
def __init__(self, uri):
|
||||
pass
|
||||
|
||||
def set_records(self, records_iterator):
|
||||
pass
|
||||
|
||||
def get_head_commit_id(self, uri, branch):
|
||||
pass
|
||||
|
||||
def set_head_commit_id(self, uri, branch, head_commit_id):
|
||||
pass
|
||||
|
||||
def get_update(self, pid):
|
||||
pass
|
||||
|
||||
def active_pids(self, pids):
|
||||
pass
|
||||
|
||||
def get_pid_update_time(self, pid):
|
||||
pass
|
||||
|
||||
def set_pid_update_time(self, pid, time):
|
||||
pass
|
||||
|
||||
def get_repo_update_time(self):
|
||||
pass
|
||||
|
||||
def set_repo_update_time(self, time):
|
||||
pass
|
||||
|
||||
|
||||
class MemcachedStorage(RuntimeStorage):
|
||||
|
||||
def __init__(self, uri):
|
||||
super(MemcachedStorage, self).__init__(uri)
|
||||
|
||||
stripped = re.sub(r'memcached:\/\/', '', uri)
|
||||
if stripped:
|
||||
storage_uri = stripped.split(',')
|
||||
self.memcached = memcache.Client(storage_uri)
|
||||
self._build_index()
|
||||
else:
|
||||
raise Exception('Invalid storage uri %s' % cfg.CONF.storage_uri)
|
||||
|
||||
def set_records(self, records_iterator):
|
||||
for record in records_iterator:
|
||||
if record['commit_id'] in self.commit_id_index:
|
||||
# update
|
||||
record_id = self.commit_id_index[record['commit_id']]
|
||||
old_record = self.memcached.get(
|
||||
self._get_record_name(record_id))
|
||||
old_record['branches'] |= record['branches']
|
||||
LOG.debug('Update record %s' % record)
|
||||
self.memcached.set(self._get_record_name(record_id),
|
||||
old_record)
|
||||
else:
|
||||
# insert record
|
||||
record_id = self._get_record_count()
|
||||
record['record_id'] = record_id
|
||||
LOG.debug('Insert new record %s' % record)
|
||||
self.memcached.set(self._get_record_name(record_id), record)
|
||||
self._set_record_count(record_id + 1)
|
||||
|
||||
self._commit_update(record_id)
|
||||
|
||||
def get_head_commit_id(self, uri, branch):
|
||||
key = str(urllib.quote_plus(uri) + ':' + branch)
|
||||
return self.memcached.get(key)
|
||||
|
||||
def set_head_commit_id(self, uri, branch, head_commit_id):
|
||||
key = str(urllib.quote_plus(uri) + ':' + branch)
|
||||
self.memcached.set(key, head_commit_id)
|
||||
|
||||
def get_update(self, pid):
|
||||
last_update = self.memcached.get('pid:%s' % pid)
|
||||
update_count = self._get_update_count()
|
||||
|
||||
self.memcached.set('pid:%s' % pid, update_count)
|
||||
self._set_pids(pid)
|
||||
|
||||
if not last_update:
|
||||
for record_id in range(0, self._get_record_count()):
|
||||
yield self.memcached.get(self._get_record_name(record_id))
|
||||
else:
|
||||
for update_id in range(last_update, update_count):
|
||||
yield self.memcached.get(self._get_record_name(
|
||||
self.memcached.get('update:%s' % update_id)))
|
||||
|
||||
def active_pids(self, pids):
|
||||
stored_pids = self.memcached.get('pids') or set()
|
||||
for pid in stored_pids:
|
||||
if pid not in pids:
|
||||
self.memcached.delete('pid:%s' % pid)
|
||||
self.memcached.delete('pid_update_time:%s' % pid)
|
||||
|
||||
self.memcached.set('pids', pids)
|
||||
|
||||
# remove unneeded updates
|
||||
min_update = self._get_update_count()
|
||||
for pid in pids:
|
||||
n = self.memcached.get('pid:%s' % pid)
|
||||
if n:
|
||||
if n < min_update:
|
||||
min_update = n
|
||||
|
||||
first_valid_update_id = self.memcached.get('first_valid_update_id')
|
||||
if not first_valid_update_id:
|
||||
first_valid_update_id = 0
|
||||
|
||||
for i in range(first_valid_update_id, min_update):
|
||||
self.memcached.delete('update:%s' % i)
|
||||
|
||||
self.memcached.set('first_valid_update_id', min_update)
|
||||
|
||||
def get_pid_update_time(self, pid):
|
||||
return self.memcached.get('pid_update_time:%s' % pid) or 0
|
||||
|
||||
def set_pid_update_time(self, pid, time):
|
||||
self.memcached.set('pid_update_time:%s' % pid, time)
|
||||
|
||||
def get_repo_update_time(self):
|
||||
return self.memcached.get('repo_update_time') or 0
|
||||
|
||||
def set_repo_update_time(self, time):
|
||||
self.memcached.set('repo_update_time', time)
|
||||
|
||||
def _get_update_count(self):
|
||||
return self.memcached.get('update:count') or 0
|
||||
|
||||
def _set_pids(self, pid):
|
||||
pids = self.memcached.get('pids') or set()
|
||||
if pid in pids:
|
||||
return
|
||||
pids.add(pid)
|
||||
self.memcached.set('pids', pids)
|
||||
|
||||
def _get_record_name(self, record_id):
|
||||
return 'record:%s' % record_id
|
||||
|
||||
def _get_record_count(self):
|
||||
return self.memcached.get('record:count') or 0
|
||||
|
||||
def _set_record_count(self, count):
|
||||
self.memcached.set('record:count', count)
|
||||
|
||||
def _get_all_records(self):
|
||||
count = self.memcached.get('record:count') or 0
|
||||
for i in range(0, count):
|
||||
yield self.memcached.get('record:%s' % i)
|
||||
|
||||
def _commit_update(self, record_id):
|
||||
count = self._get_update_count()
|
||||
self.memcached.set('update:%s' % count, record_id)
|
||||
self.memcached.set('update:count', count + 1)
|
||||
|
||||
def _build_index(self):
|
||||
self.commit_id_index = {}
|
||||
for record in self._get_all_records():
|
||||
self.commit_id_index[record['commit_id']] = record['record_id']
|
||||
|
||||
|
||||
class RuntimeStorageFactory(object):
|
||||
|
||||
@staticmethod
|
||||
def get_storage(uri):
|
||||
LOG.debug('Runtime storage is requested for uri %s' % uri)
|
||||
match = re.search(r'^memcached:\/\/', uri)
|
||||
if match:
|
||||
return MemcachedStorage(uri)
|
||||
58
stackalytics/processor/user_utils.py
Normal file
58
stackalytics/processor/user_utils.py
Normal file
@ -0,0 +1,58 @@
|
||||
# Copyright (c) 2013 Mirantis Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import datetime
|
||||
import time
|
||||
|
||||
|
||||
def normalize_user(user):
|
||||
user['emails'] = [email.lower() for email in user['emails']]
|
||||
user['launchpad_id'] = user['launchpad_id'].lower()
|
||||
|
||||
for c in user['companies']:
|
||||
end_date_numeric = 0
|
||||
if c['end_date']:
|
||||
end_date_numeric = date_to_timestamp(c['end_date'])
|
||||
c['end_date'] = end_date_numeric
|
||||
|
||||
# sort companies by end_date
|
||||
def end_date_comparator(x, y):
|
||||
if x["end_date"] == 0:
|
||||
return 1
|
||||
elif y["end_date"] == 0:
|
||||
return -1
|
||||
else:
|
||||
return cmp(x["end_date"], y["end_date"])
|
||||
|
||||
user['companies'].sort(cmp=end_date_comparator)
|
||||
return user
|
||||
|
||||
|
||||
def date_to_timestamp(d):
|
||||
if d == 'now':
|
||||
return int(time.time())
|
||||
return int(time.mktime(
|
||||
datetime.datetime.strptime(d, '%Y-%b-%d').timetuple()))
|
||||
|
||||
|
||||
def timestamp_to_week(timestamp):
|
||||
# Jan 4th 1970 is the first Sunday in the Epoch
|
||||
return (timestamp - 3 * 24 * 3600) // (7 * 24 * 3600)
|
||||
|
||||
|
||||
def week_to_date(week):
|
||||
timestamp = week * 7 * 24 * 3600 + 3 * 24 * 3600
|
||||
return (datetime.datetime.fromtimestamp(timestamp).
|
||||
strftime('%Y-%m-%d %H:%M:%S'))
|
||||
175
stackalytics/processor/vcs.py
Normal file
175
stackalytics/processor/vcs.py
Normal file
@ -0,0 +1,175 @@
|
||||
# Copyright (c) 2013 Mirantis Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import logging
|
||||
|
||||
import os
|
||||
import re
|
||||
|
||||
from oslo.config import cfg
|
||||
import sh
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Vcs(object):
|
||||
def __init__(self, repo):
|
||||
self.repo = repo
|
||||
|
||||
def fetch(self):
|
||||
pass
|
||||
|
||||
def log(self, branch, head_commit_id):
|
||||
pass
|
||||
|
||||
def get_head_commit_id(self, branch):
|
||||
pass
|
||||
|
||||
|
||||
GIT_LOG_PARAMS = [
|
||||
('commit_id', '%H'),
|
||||
('date', '%at'),
|
||||
('author', '%an'),
|
||||
('author_email', '%ae'),
|
||||
('author_email', '%ae'),
|
||||
('subject', '%s'),
|
||||
('message', '%b'),
|
||||
]
|
||||
GIT_LOG_FORMAT = ''.join([(r[0] + ':' + r[1] + '%n')
|
||||
for r in GIT_LOG_PARAMS]) + 'diff_stat:'
|
||||
DIFF_STAT_PATTERN = ('[^\d]+(\d+)\s+[^\s]*\s+changed'
|
||||
'(,\s+(\d+)\s+([^\d\s]*)\s+(\d+)?)?')
|
||||
GIT_LOG_PATTERN = re.compile(''.join([(r[0] + ':(.*?)\n')
|
||||
for r in GIT_LOG_PARAMS]) +
|
||||
'diff_stat:' + DIFF_STAT_PATTERN,
|
||||
re.DOTALL)
|
||||
|
||||
MESSAGE_PATTERNS = {
|
||||
'bug_id': re.compile('bug\s+#?([\d]{5,7})', re.IGNORECASE),
|
||||
'blueprint_id': re.compile('blueprint\s+([\w-]{6,})', re.IGNORECASE),
|
||||
'change_id': re.compile('Change-Id: (I[0-9a-f]{40})', re.IGNORECASE),
|
||||
}
|
||||
|
||||
|
||||
class Git(Vcs):
|
||||
|
||||
def __init__(self, repo):
|
||||
super(Git, self).__init__(repo)
|
||||
uri = self.repo['uri']
|
||||
match = re.search(r'([^\/]+)\.git$', uri)
|
||||
if match:
|
||||
self.module = match.group(1)
|
||||
else:
|
||||
raise Exception('Unexpected uri %s for git' % uri)
|
||||
self.release_index = {}
|
||||
|
||||
def _chdir(self):
|
||||
folder = os.path.normpath(cfg.CONF.sources_root + '/' + self.module)
|
||||
os.chdir(folder)
|
||||
|
||||
def fetch(self):
|
||||
LOG.debug('Fetching repo uri %s' % self.repo['uri'])
|
||||
|
||||
folder = os.path.normpath(cfg.CONF.sources_root + '/' + self.module)
|
||||
|
||||
if not os.path.exists(folder):
|
||||
os.chdir(cfg.CONF.sources_root)
|
||||
sh.git('clone', '%s' % self.repo['uri'])
|
||||
os.chdir(folder)
|
||||
else:
|
||||
self._chdir()
|
||||
sh.git('pull', 'origin')
|
||||
|
||||
for release in self.repo['releases']:
|
||||
release_name = release['release_name'].lower()
|
||||
tag_range = release['tag_from'] + '..' + release['tag_to']
|
||||
git_log_iterator = sh.git('log', '--pretty=%H', tag_range,
|
||||
_tty_out=False)
|
||||
for commit_id in git_log_iterator:
|
||||
self.release_index[commit_id.strip()] = release_name
|
||||
|
||||
def log(self, branch, head_commit_id):
|
||||
LOG.debug('Parsing git log for repo uri %s' % self.repo['uri'])
|
||||
|
||||
self._chdir()
|
||||
sh.git('checkout', '%s' % branch)
|
||||
commit_range = 'HEAD'
|
||||
if head_commit_id:
|
||||
commit_range = head_commit_id + '..HEAD'
|
||||
output = sh.git('log', '--pretty=%s' % GIT_LOG_FORMAT, '--shortstat',
|
||||
'-M', '--no-merges', commit_range, _tty_out=False)
|
||||
|
||||
for rec in re.finditer(GIT_LOG_PATTERN, str(output)):
|
||||
i = 1
|
||||
commit = {}
|
||||
for param in GIT_LOG_PARAMS:
|
||||
commit[param[0]] = unicode(rec.group(i), 'utf8')
|
||||
i += 1
|
||||
|
||||
commit['files_changed'] = int(rec.group(i))
|
||||
i += 1
|
||||
lines_changed_group = rec.group(i)
|
||||
i += 1
|
||||
lines_changed = rec.group(i)
|
||||
i += 1
|
||||
deleted_or_inserted = rec.group(i)
|
||||
i += 1
|
||||
lines_deleted = rec.group(i)
|
||||
i += 1
|
||||
|
||||
if lines_changed_group: # there inserted or deleted lines
|
||||
if not lines_deleted:
|
||||
if deleted_or_inserted[0] == 'd': # deleted
|
||||
lines_deleted = lines_changed
|
||||
lines_changed = 0
|
||||
|
||||
commit['lines_added'] = int(lines_changed or 0)
|
||||
commit['lines_deleted'] = int(lines_deleted or 0)
|
||||
|
||||
for key in MESSAGE_PATTERNS:
|
||||
match = re.search(MESSAGE_PATTERNS[key], commit['message'])
|
||||
if match:
|
||||
commit[key] = match.group(1)
|
||||
else:
|
||||
commit[key] = None
|
||||
|
||||
commit['date'] = int(commit['date'])
|
||||
commit['module'] = self.module
|
||||
commit['branches'] = set([branch])
|
||||
if commit['commit_id'] in self.release_index:
|
||||
commit['release'] = self.release_index[commit['commit_id']]
|
||||
else:
|
||||
commit['release'] = None
|
||||
|
||||
yield commit
|
||||
|
||||
def get_head_commit_id(self, branch):
|
||||
LOG.debug('Get head commit for repo uri %s' % self.repo['uri'])
|
||||
|
||||
self._chdir()
|
||||
sh.git('checkout', '%s' % branch)
|
||||
return str(sh.git('rev-parse', 'HEAD')).strip()
|
||||
|
||||
|
||||
class VcsFactory(object):
|
||||
|
||||
@staticmethod
|
||||
def get_vcs(repo):
|
||||
uri = repo['uri']
|
||||
LOG.debug('Factory is asked for Vcs uri %s' % uri)
|
||||
match = re.search(r'\.git$', uri)
|
||||
if match:
|
||||
return Git(repo)
|
||||
#todo others vcs to be implemented
|
||||
@ -7,7 +7,7 @@ hacking>=0.5.3,<0.6
|
||||
coverage
|
||||
discover
|
||||
fixtures>=0.3.12
|
||||
mox
|
||||
mock
|
||||
python-subunit
|
||||
testrepository>=0.0.13
|
||||
testtools>=0.9.22
|
||||
|
||||
231
tests/unit/test_commit_processor.py
Normal file
231
tests/unit/test_commit_processor.py
Normal file
@ -0,0 +1,231 @@
|
||||
from launchpadlib import launchpad
|
||||
import mock
|
||||
from oslo.config import cfg
|
||||
import testtools
|
||||
|
||||
from stackalytics.processor import commit_processor
|
||||
from stackalytics.processor import persistent_storage
|
||||
|
||||
|
||||
class TestCommitProcessor(testtools.TestCase):
|
||||
def setUp(self):
|
||||
super(TestCommitProcessor, self).setUp()
|
||||
|
||||
p_storage = mock.Mock(persistent_storage.PersistentStorage)
|
||||
p_storage.get_companies = mock.Mock(return_value=[
|
||||
{
|
||||
'company_name': 'SuperCompany',
|
||||
'domains': ['super.com', 'super.no']
|
||||
},
|
||||
{
|
||||
"domains": ["nec.com", "nec.co.jp"],
|
||||
"company_name": "NEC"
|
||||
},
|
||||
{
|
||||
'company_name': '*independent',
|
||||
'domains': ['']
|
||||
},
|
||||
])
|
||||
self.user = {
|
||||
'launchpad_id': 'john_doe', 'user_name': 'John Doe',
|
||||
'emails': ['johndoe@gmail.com', 'jdoe@super.no'],
|
||||
'companies': [
|
||||
{'company_name': '*independent',
|
||||
'end_date': 1234567890},
|
||||
{'company_name': 'SuperCompany',
|
||||
'end_date': 0},
|
||||
]
|
||||
}
|
||||
p_storage.get_users = mock.Mock(return_value=[
|
||||
self.user,
|
||||
])
|
||||
|
||||
self.persistent_storage = p_storage
|
||||
self.commit_processor = commit_processor.CachedProcessor(p_storage)
|
||||
self.launchpad_patch = mock.patch('launchpadlib.launchpad.Launchpad')
|
||||
self.launchpad_patch.start()
|
||||
cfg.CONF = mock.MagicMock()
|
||||
|
||||
def tearDown(self):
|
||||
super(TestCommitProcessor, self).tearDown()
|
||||
self.launchpad_patch.stop()
|
||||
|
||||
def test_get_company_by_email_mapped(self):
|
||||
email = 'jdoe@super.no'
|
||||
res = self.commit_processor._get_company_by_email(email)
|
||||
self.assertEquals('SuperCompany', res)
|
||||
|
||||
def test_get_company_by_email_with_long_suffix_mapped(self):
|
||||
email = 'man@mxw.nes.nec.co.jp'
|
||||
res = self.commit_processor._get_company_by_email(email)
|
||||
self.assertEquals('NEC', res)
|
||||
|
||||
def test_get_company_by_email_with_long_suffix_mapped_2(self):
|
||||
email = 'man@mxw.nes.nec.com'
|
||||
res = self.commit_processor._get_company_by_email(email)
|
||||
self.assertEquals('NEC', res)
|
||||
|
||||
def test_get_company_by_email_not_mapped(self):
|
||||
email = 'foo@boo.com'
|
||||
res = self.commit_processor._get_company_by_email(email)
|
||||
self.assertEquals(None, res)
|
||||
|
||||
def test_update_commit_existing_user(self):
|
||||
commit = {
|
||||
'author_email': 'johndoe@gmail.com',
|
||||
'date': 1999999999,
|
||||
}
|
||||
self.commit_processor._update_commit_with_user_data(commit)
|
||||
|
||||
self.assertEquals('SuperCompany', commit['company_name'])
|
||||
self.assertEquals('john_doe', commit['launchpad_id'])
|
||||
|
||||
def test_update_commit_existing_user_old_job(self):
|
||||
commit = {
|
||||
'author_email': 'johndoe@gmail.com',
|
||||
'date': 1000000000,
|
||||
}
|
||||
self.commit_processor._update_commit_with_user_data(commit)
|
||||
|
||||
self.assertEquals('*independent', commit['company_name'])
|
||||
self.assertEquals('john_doe', commit['launchpad_id'])
|
||||
|
||||
def test_update_commit_existing_user_new_email_known_company(self):
|
||||
"""
|
||||
User is known to LP, his email is new to us, and maps to other company
|
||||
Should return other company instead of those mentioned in user db
|
||||
"""
|
||||
email = 'johndoe@nec.co.jp'
|
||||
commit = {
|
||||
'author_email': email,
|
||||
'date': 1999999999,
|
||||
}
|
||||
lp_mock = mock.MagicMock()
|
||||
launchpad.Launchpad.login_anonymously = mock.Mock(return_value=lp_mock)
|
||||
lp_profile = mock.Mock()
|
||||
lp_profile.name = 'john_doe'
|
||||
lp_mock.people.getByEmail = mock.Mock(return_value=lp_profile)
|
||||
user = self.user.copy()
|
||||
# tell storage to return existing user
|
||||
self.persistent_storage.get_users.return_value = [user]
|
||||
|
||||
self.commit_processor._update_commit_with_user_data(commit)
|
||||
|
||||
self.persistent_storage.update_user.assert_called_once_with(user)
|
||||
lp_mock.people.getByEmail.assert_called_once_with(email=email)
|
||||
self.assertIn(email, user['emails'])
|
||||
self.assertEquals('NEC', commit['company_name'])
|
||||
self.assertEquals('john_doe', commit['launchpad_id'])
|
||||
|
||||
def test_update_commit_existing_user_new_email_unknown_company(self):
|
||||
"""
|
||||
User is known to LP, but his email is new to us. Should match
|
||||
the user and return current company
|
||||
"""
|
||||
email = 'johndoe@yahoo.com'
|
||||
commit = {
|
||||
'author_email': email,
|
||||
'date': 1999999999,
|
||||
}
|
||||
lp_mock = mock.MagicMock()
|
||||
launchpad.Launchpad.login_anonymously = mock.Mock(return_value=lp_mock)
|
||||
lp_profile = mock.Mock()
|
||||
lp_profile.name = 'john_doe'
|
||||
lp_mock.people.getByEmail = mock.Mock(return_value=lp_profile)
|
||||
user = self.user.copy()
|
||||
# tell storage to return existing user
|
||||
self.persistent_storage.get_users.return_value = [user]
|
||||
|
||||
self.commit_processor._update_commit_with_user_data(commit)
|
||||
|
||||
self.persistent_storage.update_user.assert_called_once_with(user)
|
||||
lp_mock.people.getByEmail.assert_called_once_with(email=email)
|
||||
self.assertIn(email, user['emails'])
|
||||
self.assertEquals('SuperCompany', commit['company_name'])
|
||||
self.assertEquals('john_doe', commit['launchpad_id'])
|
||||
|
||||
def test_update_commit_new_user(self):
|
||||
"""
|
||||
User is known to LP, but new to us
|
||||
Should add new user and set company depending on email
|
||||
"""
|
||||
email = 'smith@nec.com'
|
||||
commit = {
|
||||
'author_email': email,
|
||||
'date': 1999999999,
|
||||
}
|
||||
lp_mock = mock.MagicMock()
|
||||
launchpad.Launchpad.login_anonymously = mock.Mock(return_value=lp_mock)
|
||||
lp_profile = mock.Mock()
|
||||
lp_profile.name = 'smith'
|
||||
lp_profile.display_name = 'Smith'
|
||||
lp_mock.people.getByEmail = mock.Mock(return_value=lp_profile)
|
||||
self.persistent_storage.get_users.return_value = []
|
||||
|
||||
self.commit_processor._update_commit_with_user_data(commit)
|
||||
|
||||
lp_mock.people.getByEmail.assert_called_once_with(email=email)
|
||||
self.assertEquals('NEC', commit['company_name'])
|
||||
self.assertEquals('smith', commit['launchpad_id'])
|
||||
|
||||
def test_update_commit_new_user_unknown_to_lb(self):
|
||||
"""
|
||||
User is new to us and not known to LP
|
||||
Should set user name and empty LPid
|
||||
"""
|
||||
email = 'inkognito@avs.com'
|
||||
commit = {
|
||||
'author_email': email,
|
||||
'date': 1999999999,
|
||||
}
|
||||
lp_mock = mock.MagicMock()
|
||||
launchpad.Launchpad.login_anonymously = mock.Mock(return_value=lp_mock)
|
||||
lp_mock.people.getByEmail = mock.Mock(return_value=None)
|
||||
self.persistent_storage.get_users.return_value = []
|
||||
|
||||
self.commit_processor._update_commit_with_user_data(commit)
|
||||
|
||||
lp_mock.people.getByEmail.assert_called_once_with(email=email)
|
||||
self.assertEquals('*independent', commit['company_name'])
|
||||
self.assertEquals(None, commit['launchpad_id'])
|
||||
|
||||
def test_update_commit_new_user_lb_raises_error(self):
|
||||
"""
|
||||
LP raises error during getting user info
|
||||
"""
|
||||
email = 'smith@avs.com'
|
||||
commit = {
|
||||
'author_email': email,
|
||||
'date': 1999999999,
|
||||
}
|
||||
lp_mock = mock.MagicMock()
|
||||
launchpad.Launchpad.login_anonymously = mock.Mock(return_value=lp_mock)
|
||||
lp_mock.people.getByEmail = mock.Mock(return_value=None,
|
||||
side_effect=Exception)
|
||||
self.persistent_storage.get_users.return_value = []
|
||||
|
||||
self.commit_processor._update_commit_with_user_data(commit)
|
||||
|
||||
lp_mock.people.getByEmail.assert_called_once_with(email=email)
|
||||
self.assertEquals('*independent', commit['company_name'])
|
||||
self.assertEquals(None, commit['launchpad_id'])
|
||||
|
||||
def test_update_commit_invalid_email(self):
|
||||
"""
|
||||
User's email is malformed
|
||||
"""
|
||||
email = 'error.root'
|
||||
commit = {
|
||||
'author_email': email,
|
||||
'date': 1999999999,
|
||||
}
|
||||
lp_mock = mock.MagicMock()
|
||||
launchpad.Launchpad.login_anonymously = mock.Mock(return_value=lp_mock)
|
||||
lp_mock.people.getByEmail = mock.Mock(return_value=None)
|
||||
self.persistent_storage.get_users.return_value = []
|
||||
|
||||
self.commit_processor._update_commit_with_user_data(commit)
|
||||
|
||||
self.assertEquals(0, lp_mock.people.getByEmail.called)
|
||||
self.assertEquals('*independent', commit['company_name'])
|
||||
self.assertEquals(None, commit['launchpad_id'])
|
||||
@ -1,22 +0,0 @@
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from dashboard import dashboard
|
||||
|
||||
|
||||
class DashboardTestCase(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.db_fd, dashboard.app.config['DATABASE'] = tempfile.mkstemp()
|
||||
dashboard.app.config['TESTING'] = True
|
||||
self.app = dashboard.app.test_client()
|
||||
# dashboard.init_db()
|
||||
|
||||
def tearDown(self):
|
||||
os.close(self.db_fd)
|
||||
os.unlink(dashboard.app.config['DATABASE'])
|
||||
|
||||
def test_home_page(self):
|
||||
rv = self.app.get('/')
|
||||
assert rv.status_code == 200
|
||||
110
tests/unit/test_vcs.py
Normal file
110
tests/unit/test_vcs.py
Normal file
@ -0,0 +1,110 @@
|
||||
import mock
|
||||
import os
|
||||
import testtools
|
||||
|
||||
from oslo.config import cfg
|
||||
|
||||
from stackalytics.processor import vcs
|
||||
|
||||
|
||||
class TestVcsProcessor(testtools.TestCase):
|
||||
def setUp(self):
|
||||
super(TestVcsProcessor, self).setUp()
|
||||
|
||||
self.repo = {
|
||||
'uri': 'git://github.com/dummy.git',
|
||||
'releases': []
|
||||
}
|
||||
self.git = vcs.Git(self.repo)
|
||||
cfg.CONF.sources_root = ''
|
||||
os.chdir = mock.Mock()
|
||||
|
||||
def test_git_log(self):
|
||||
with mock.patch('sh.git') as git_mock:
|
||||
git_mock.return_value = '''
|
||||
commit_id:b5a416ac344160512f95751ae16e6612aefd4a57
|
||||
date:1369119386
|
||||
author:Akihiro MOTOKI
|
||||
author_email:motoki@da.jp.nec.com
|
||||
author_email:motoki@da.jp.nec.com
|
||||
subject:Remove class-based import in the code repo
|
||||
message:Fixes bug 1167901
|
||||
|
||||
This commit also removes backslashes for line break.
|
||||
|
||||
Change-Id: Id26fdfd2af4862652d7270aec132d40662efeb96
|
||||
|
||||
diff_stat:
|
||||
|
||||
21 files changed, 340 insertions(+), 408 deletions(-)
|
||||
commit_id:5be031f81f76d68c6e4cbaad2247044aca179843
|
||||
date:1370975889
|
||||
author:Monty Taylor
|
||||
author_email:mordred@inaugust.com
|
||||
author_email:mordred@inaugust.com
|
||||
subject:Remove explicit distribute depend.
|
||||
message:Causes issues with the recent re-merge with setuptools. Advice from
|
||||
upstream is to stop doing explicit depends.
|
||||
|
||||
Change-Id: I70638f239794e78ba049c60d2001190910a89c90
|
||||
|
||||
diff_stat:
|
||||
|
||||
1 file changed, 1 deletion(-)
|
||||
commit_id:92811c76f3a8308b36f81e61451ec17d227b453b
|
||||
date:1369831203
|
||||
author:Mark McClain
|
||||
author_email:mark.mcclain@dreamhost.com
|
||||
author_email:mark.mcclain@dreamhost.com
|
||||
subject:add readme for 2.2.2
|
||||
message:Change-Id: Id32a4a72ec1d13992b306c4a38e73605758e26c7
|
||||
|
||||
diff_stat:
|
||||
|
||||
1 file changed, 8 insertions(+)
|
||||
commit_id:92811c76f3a8308b36f81e61451ec17d227b453b
|
||||
date:1369831203
|
||||
author:John Doe
|
||||
author_email:john.doe@dreamhost.com
|
||||
author_email:john.doe@dreamhost.com
|
||||
subject:add readme for 2.2.2
|
||||
message:Change-Id: Id32a4a72ec1d13992b306c4a38e73605758e26c7
|
||||
|
||||
diff_stat:
|
||||
|
||||
0 files changed
|
||||
commit_id:92811c76f3a8308b36f81e61451ec17d227b453b
|
||||
date:1369831203
|
||||
author:Doug Hoffner
|
||||
author_email:mark.mcclain@dreamhost.com
|
||||
author_email:mark.mcclain@dreamhost.com
|
||||
subject:add readme for 2.2.2
|
||||
message:Change-Id: Id32a4a72ec1d13992b306c4a38e73605758e26c7
|
||||
|
||||
diff_stat:
|
||||
|
||||
0 files changed, 0 insertions(+), 0 deletions(-)
|
||||
'''
|
||||
|
||||
commits = list(self.git.log('dummy', 'dummy'))
|
||||
self.assertEquals(5, len(commits))
|
||||
|
||||
self.assertEquals(21, commits[0]['files_changed'])
|
||||
self.assertEquals(340, commits[0]['lines_added'])
|
||||
self.assertEquals(408, commits[0]['lines_deleted'])
|
||||
|
||||
self.assertEquals(1, commits[1]['files_changed'])
|
||||
self.assertEquals(0, commits[1]['lines_added'])
|
||||
self.assertEquals(1, commits[1]['lines_deleted'])
|
||||
|
||||
self.assertEquals(1, commits[2]['files_changed'])
|
||||
self.assertEquals(8, commits[2]['lines_added'])
|
||||
self.assertEquals(0, commits[2]['lines_deleted'])
|
||||
|
||||
self.assertEquals(0, commits[3]['files_changed'])
|
||||
self.assertEquals(0, commits[3]['lines_added'])
|
||||
self.assertEquals(0, commits[3]['lines_deleted'])
|
||||
|
||||
self.assertEquals(0, commits[4]['files_changed'])
|
||||
self.assertEquals(0, commits[4]['lines_added'])
|
||||
self.assertEquals(0, commits[4]['lines_deleted'])
|
||||
5
tox.ini
5
tox.ini
@ -1,5 +1,5 @@
|
||||
[tox]
|
||||
envlist = py27,pep8
|
||||
envlist = py26,py27,pep8
|
||||
|
||||
[testenv]
|
||||
setenv = VIRTUAL_ENV={envdir}
|
||||
@ -26,7 +26,8 @@ downloadcache = ~/cache/pip
|
||||
|
||||
[flake8]
|
||||
# E125 continuation line does not distinguish itself from next logical line
|
||||
ignore = E125
|
||||
# H404 multi line docstring should start with a summary
|
||||
ignore = E125,H404
|
||||
show-source = true
|
||||
builtins = _
|
||||
exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,tools,pycvsanaly2
|
||||
|
||||
Loading…
Reference in New Issue
Block a user