x/stackalytics
Code Issues Proposed changes

Poll gerrit only for projects that have it

Browse Source 
Currently Stackalytics polls gerrit for all projects,
even for external non-OpenStack. With this patch communication
with gerrit is optimized and only hosted projects are queried.
The patch should decrease number of gerrit communication
errors.

Change-Id: I738cac6eb92615c5fe5d9e1bf1b878197d78876f
This commit is contained in:
Ilya Shakhat 2016-07-06 11:23:50 +03:00
parent f7b268fd4c
commit 6e8d1fab8a
3 changed files with 18 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
stackalytics/processor/default_data_processor.py
View File

@ -94,7 +94,8 @@ def _retrieve_project_list_from_gerrit(project_source):
'module': name,
'organization': org,
'uri': repo_uri,
'releases': []
'releases': [],
'has_gerrit': True,
}
@ -147,7 +148,14 @@ def _update_project_list(default_data):
repos = _retrieve_project_list_from_sources(
default_data['project_sources'])
if repos:
default_data['repos'] += [r for r in repos
# update pre-configured
repos_dict = dict((r['uri'], r) for r in repos)
for r in default_data['repos']:
if r['uri'] in repos_dict:
r.update(repos_dict[r['uri']])
# update default data
default_data['repos'] += [r for r in repos_dict.values()
if r['uri'] not in configured_repos]
default_data['module_groups'] += _create_module_groups_for_project_sources(

3
stackalytics/processor/main.py
View File

@ -131,6 +131,9 @@ def _process_repo(repo, runtime_storage_inst, record_processor_inst,
last_id = vcs_inst.get_last_id(branch)
runtime_storage_inst.set_by_key(vcs_key, last_id)
if 'has_gerrit' not in repo:
continue # do not poll reviews for those that do not have them
LOG.info('Processing reviews for repo: %s, branch: %s', uri, branch)
rcs_key = 'rcs:%s:%s' % (quoted_uri, branch)

5
stackalytics/tests/unit/test_default_data_processor.py
View File

@ -65,6 +65,7 @@ class TestDefaultDataProcessor(testtools.TestCase):
'uri': 'git://git.openstack.org/openstack/nova',
'organization': 'openstack'},
{'module': 'qa', 'uri': 'git://git.openstack.org/openstack/qa',
'has_gerrit': True,
'organization': 'openstack'},
]
dd = {
@ -88,6 +89,10 @@ class TestDefaultDataProcessor(testtools.TestCase):
self.assertIn('nova', set([r['module'] for r in dd['repos']]))
self.assertIn('tux', set([r['module'] for r in dd['repos']]))
self.assertIn('has_gerrit', dd['repos'][0])
self.assertNotIn('has_gerrit', dd['repos'][1])
self.assertNotIn('has_gerrit', dd['repos'][2])
self.assertEqual(2, len(dd['module_groups']))
self.assertIn({'id': 'openstack',
'module_group_name': 'openstack',