x/stackalytics
Code Issues Proposed changes

Make configurable number of member profiles to look ahead

Browse Source 
Introduced config param 'members_look_ahead' with default value 250

Closes bug 1361927

Change-Id: Ia4c029126efe0044096768dc27e19e6345b7c38d
This commit is contained in:
Ilya Shakhat 2014-09-02 16:38:18 +04:00
parent c2f873b669
commit bfb56d28c2
4 changed files with 9 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
etc/stackalytics.conf
View File

@ -58,3 +58,6 @@
# Name of file to store python profiler data. This option works for dashboard only
# collect_profiler_stats =
# How many member profiles to look ahead after the last
# members_look_ahead = 250

2
stackalytics/processor/config.py
View File

@ -65,4 +65,6 @@ OPTS = [
cfg.StrOpt('collect-profiler-stats',
help='Name of file to store python profiler data. This option '
'works for dashboard only'),
cfg.IntOpt('members-look-ahead', default=250,
help='How many member profiles to look ahead after the last'),
]

3
stackalytics/processor/main.py
View File

@ -175,7 +175,8 @@ def _process_mail_list(uri, runtime_storage_inst, record_processor_inst):
def _process_member_list(uri, runtime_storage_inst, record_processor_inst):
member_iterator = mps.log(uri, runtime_storage_inst,
cfg.CONF.days_to_update_members)
cfg.CONF.days_to_update_members,
cfg.CONF.members_look_ahead)
member_iterator_typed = _record_typer(member_iterator, 'member')
processed_member_iterator = record_processor_inst.process(
member_iterator_typed)

6
stackalytics/processor/mps.py
View File

@ -29,8 +29,6 @@ NAME_AND_DATE_PATTERN = r'<h3>(?P<member_name>[^<]*)[\s\S]*?' \
COMPANY_PATTERN = r'<strong>Date\sJoined[\s\S]*?<b>(?P<company_draft>[^<]*)' \
r'[\s\S]*?From\s(?P<date_from>[\s\S]*?)\(Current\)'
CNT_EMPTY_MEMBERS = 50
def _convert_str_fields_to_unicode(result):
for field, value in six.iteritems(result):
@ -69,7 +67,7 @@ def _retrieve_member(uri, member_id, html_parser):
return member
def log(uri, runtime_storage_inst, days_to_update_members):
def log(uri, runtime_storage_inst, days_to_update_members, members_look_ahead):
LOG.debug('Retrieving new openstack.org members')
last_update_members_date = runtime_storage_inst.get_by_key(
@ -90,7 +88,7 @@ def log(uri, runtime_storage_inst, days_to_update_members):
cur_index = last_member_index + 1
html_parser = six.moves.html_parser.HTMLParser()
while cnt_empty < CNT_EMPTY_MEMBERS:
while cnt_empty < members_look_ahead:
profile_uri = uri + str(cur_index)
member = _retrieve_member(profile_uri, str(cur_index), html_parser)