diff --git a/doc/source/tools/stackalytics-processor.txt b/doc/source/tools/stackalytics-processor.txt index e1180acc0..b747b7e01 100644 --- a/doc/source/tools/stackalytics-processor.txt +++ b/doc/source/tools/stackalytics-processor.txt @@ -3,6 +3,7 @@ usage: stackalytics-processor [-h] [--config-dir DIR] [--config-file PATH] [--days_to_update_members DAYS_TO_UPDATE_MEMBERS] [--debug] [--default-data-uri DEFAULT_DATA_URI] [--driverlog-data-uri DRIVERLOG_DATA_URI] + [--fetching-user-source FETCHING_USER_SOURCE] [--gerrit-retry GERRIT_RETRY] [--git-base-uri GIT_BASE_URI] [--log-config-append PATH] @@ -44,6 +45,8 @@ optional arguments: file:///path/to/default_data.json --driverlog-data-uri DRIVERLOG_DATA_URI URI for default data + --fetching-user-source FETCHING_USER_SOURCE + Source for fetching user profiles --gerrit-retry GERRIT_RETRY How many times to retry after Gerrit errors --git-base-uri GIT_BASE_URI diff --git a/etc/stackalytics.conf b/etc/stackalytics.conf index 321b5d2a5..ef8f51236 100644 --- a/etc/stackalytics.conf +++ b/etc/stackalytics.conf @@ -147,6 +147,10 @@ # URI of translation team data (string value) #translation_team_uri = https://git.openstack.org/cgit/openstack/i18n/plain/tools/zanata/translation_team.yaml +# Source for fetching user profiles (string value) +# Allowed values: launchpad, +#fetching_user_source = launchpad + # How many member profiles to look ahead after the last (integer value) #members_look_ahead = 250 diff --git a/stackalytics/processor/config.py b/stackalytics/processor/config.py index 18e922554..c7b630bd9 100644 --- a/stackalytics/processor/config.py +++ b/stackalytics/processor/config.py @@ -54,6 +54,9 @@ PROCESSOR_OPTS = [ default='https://git.openstack.org/cgit/openstack/i18n/' 'plain/tools/zanata/translation_team.yaml', help='URI of translation team data'), + cfg.StrOpt("fetching-user-source", default='launchpad', + choices=['launchpad', ''], + help="Source for fetching user profiles"), cfg.IntOpt('members-look-ahead', default=250, help='How many member profiles to look ahead after the last'), cfg.IntOpt('read-timeout', default=120, diff --git a/stackalytics/processor/record_processor.py b/stackalytics/processor/record_processor.py index 9f139dd4d..f801cba9f 100644 --- a/stackalytics/processor/record_processor.py +++ b/stackalytics/processor/record_processor.py @@ -19,6 +19,7 @@ import copy import functools import time +from oslo_config import cfg from oslo_log import log as logging import six @@ -27,6 +28,7 @@ from stackalytics.processor import user_processor from stackalytics.processor import utils +CONF = cfg.CONF LOG = logging.getLogger(__name__) @@ -232,6 +234,9 @@ class RecordProcessor(object): self.runtime_storage_inst, u) return merged_user + def _need_to_fetch_launchpad(self): + return CONF.fetching_user_source == 'launchpad' + def update_user(self, record): email = record.get('author_email') user_e = user_processor.load_user( @@ -239,8 +244,8 @@ class RecordProcessor(object): user_name = record.get('author_name') launchpad_id = record.get('launchpad_id') - if (email and (not user_e) and (not launchpad_id) and - (not user_e.get('launchpad_id'))): + if (self._need_to_fetch_launchpad() and email and (not user_e) and + (not launchpad_id) and (not user_e.get('launchpad_id'))): # query LP launchpad_id, lp_user_name = self._get_lp_info(email) if lp_user_name: @@ -250,8 +255,8 @@ class RecordProcessor(object): if gerrit_id: user_g = user_processor.load_user( self.runtime_storage_inst, gerrit_id=gerrit_id) or {} - if ((not user_g) and (not launchpad_id) and - (not user_e.get('launchpad_id'))): + if (self._need_to_fetch_launchpad() and (not user_g) and + (not launchpad_id) and (not user_e.get('launchpad_id'))): # query LP guessed_lp_id = gerrit_id lp_user_name = self._get_lp_user_name(guessed_lp_id) @@ -264,8 +269,8 @@ class RecordProcessor(object): if zanata_id: user_z = user_processor.load_user( self.runtime_storage_inst, zanata_id=zanata_id) or {} - if ((not user_z) and (not launchpad_id) and - (not user_e.get('launchpad_id'))): + if (self._need_to_fetch_launchpad() and (not user_z) and + (not launchpad_id) and (not user_e.get('launchpad_id'))): # query LP guessed_lp_id = zanata_id user_name = self._get_lp_user_name(guessed_lp_id) @@ -290,7 +295,7 @@ class RecordProcessor(object): [user_e, user_l, user_g, user_z, user]) else: # create new - if not user_name: + if (self._need_to_fetch_launchpad() and not user_name): user_name = self._get_lp_user_name(launchpad_id) if user_name: user['user_name'] = user_name diff --git a/stackalytics/tests/unit/test_record_processor.py b/stackalytics/tests/unit/test_record_processor.py index 91d33cca7..ef339aa33 100644 --- a/stackalytics/tests/unit/test_record_processor.py +++ b/stackalytics/tests/unit/test_record_processor.py @@ -16,15 +16,19 @@ import time import mock +from oslo_config import cfg import six import testtools +from stackalytics.processor import config from stackalytics.processor import record_processor from stackalytics.processor import runtime_storage from stackalytics.processor import user_processor from stackalytics.processor import utils +CONF = cfg.CONF + RELEASES = [ { 'release_name': 'prehistory', @@ -67,6 +71,7 @@ class TestRecordProcessor(testtools.TestCase): self.lp_profile_by_email = ( self.lp_profile_by_email_patch.start()) self.lp_profile_by_email.return_value = None + CONF.register_opts(config.CONNECTION_OPTS + config.PROCESSOR_OPTS) def tearDown(self): super(TestRecordProcessor, self).tearDown()