diff --git a/stackalytics/processor/default_data_processor.py b/stackalytics/processor/default_data_processor.py index 71324226d..aef9fd1e4 100644 --- a/stackalytics/processor/default_data_processor.py +++ b/stackalytics/processor/default_data_processor.py @@ -94,7 +94,8 @@ def _retrieve_project_list_from_gerrit(project_source): 'module': name, 'organization': org, 'uri': repo_uri, - 'releases': [] + 'releases': [], + 'has_gerrit': True, } @@ -147,7 +148,14 @@ def _update_project_list(default_data): repos = _retrieve_project_list_from_sources( default_data['project_sources']) if repos: - default_data['repos'] += [r for r in repos + # update pre-configured + repos_dict = dict((r['uri'], r) for r in repos) + for r in default_data['repos']: + if r['uri'] in repos_dict: + r.update(repos_dict[r['uri']]) + + # update default data + default_data['repos'] += [r for r in repos_dict.values() if r['uri'] not in configured_repos] default_data['module_groups'] += _create_module_groups_for_project_sources( diff --git a/stackalytics/processor/main.py b/stackalytics/processor/main.py index 5ab8a258f..2630d4202 100644 --- a/stackalytics/processor/main.py +++ b/stackalytics/processor/main.py @@ -131,6 +131,9 @@ def _process_repo(repo, runtime_storage_inst, record_processor_inst, last_id = vcs_inst.get_last_id(branch) runtime_storage_inst.set_by_key(vcs_key, last_id) + if 'has_gerrit' not in repo: + continue # do not poll reviews for those that do not have them + LOG.info('Processing reviews for repo: %s, branch: %s', uri, branch) rcs_key = 'rcs:%s:%s' % (quoted_uri, branch) diff --git a/stackalytics/tests/unit/test_default_data_processor.py b/stackalytics/tests/unit/test_default_data_processor.py index 681c4be62..2ad50dd19 100644 --- a/stackalytics/tests/unit/test_default_data_processor.py +++ b/stackalytics/tests/unit/test_default_data_processor.py @@ -65,6 +65,7 @@ class TestDefaultDataProcessor(testtools.TestCase): 'uri': 'git://git.openstack.org/openstack/nova', 'organization': 'openstack'}, {'module': 'qa', 'uri': 'git://git.openstack.org/openstack/qa', + 'has_gerrit': True, 'organization': 'openstack'}, ] dd = { @@ -88,6 +89,10 @@ class TestDefaultDataProcessor(testtools.TestCase): self.assertIn('nova', set([r['module'] for r in dd['repos']])) self.assertIn('tux', set([r['module'] for r in dd['repos']])) + self.assertIn('has_gerrit', dd['repos'][0]) + self.assertNotIn('has_gerrit', dd['repos'][1]) + self.assertNotIn('has_gerrit', dd['repos'][2]) + self.assertEqual(2, len(dd['module_groups'])) self.assertIn({'id': 'openstack', 'module_group_name': 'openstack',