openstack/election
Code Issues Proposed changes

Move general purpose functions out of owners

Browse Source 
Change-Id: I17c16d7464647f8d81cdb7cf45dba8e63be5a005
This commit is contained in:
Tony Breeds 2018-04-12 13:50:57 +10:00
parent af28fc385c
commit 7a802b65ad
2 changed files with 73 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
openstack_election/owners.py
View File

@ -21,13 +21,12 @@ from __future__ import print_function
import csv
import datetime
import json
import os
import sys
import requests
import yaml
from openstack_election import utils
try:
from string import maketrans
except ImportError: # Python3
@ -91,73 +90,6 @@ def date_merged(change, after=None, before=None):
return date
def requester(url, params={}, headers={}):
"""A requests wrapper to consistently retry HTTPS queries"""
# Try up to 3 times
retry = requests.Session()
retry.mount("https://", requests.adapters.HTTPAdapter(max_retries=3))
return retry.get(url=url, params=params, headers=headers)
def decode_json(raw):
"""Trap JSON decoding failures and provide more detailed errors"""
# Gerrit's REST API prepends a JSON-breaker to avoid XSS vulnerabilities
if raw.text.startswith(")]}'"):
trimmed = raw.text[4:]
else:
trimmed = raw.text
# Try to decode and bail with much detail if it fails
try:
decoded = json.loads(trimmed)
except Exception:
print('\nrequest returned %s error to query:\n\n %s\n'
'\nwith detail:\n\n %s\n' % (raw, raw.url, trimmed),
file=sys.stderr)
raise
return decoded
def query_gerrit(method, params={}):
"""Query the Gerrit REST API"""
# The base URL to Gerrit REST API
GERRIT_API_URL = 'https://review.openstack.org/'
raw = requester(GERRIT_API_URL + method, params=params,
headers={'Accept': 'application/json'})
return decode_json(raw)
def get_from_cgit(project, obj, params={}):
"""Retrieve a file from the cgit interface"""
url = 'http://git.openstack.org/cgit/' + project + '/plain/' + obj
raw = requester(url, params=params,
headers={'Accept': 'application/json'})
return yaml.safe_load(raw.text)
def lookup_member(email):
"""A requests wrapper to querying the OSF member directory API"""
# The OpenStack foundation member directory lookup API endpoint
MEMBER_LOOKUP_URL = 'https://openstackid-resources.openstack.org/'
# URL pattern for querying foundation members by E-mail address
raw = requester(MEMBER_LOOKUP_URL + '/api/public/v1/members',
params={'filter[]': [
'group_slug==foundation-members',
'email==' + email,
]},
headers={'Accept': 'application/json'},
)
return decode_json(raw)
def main(options):
"""The giant pile of spaghetti which does everything else"""
@ -241,17 +173,17 @@ def main(options):
# TODO(fungi): make this a configurable option so that you can
# for example supply a custom project list for running elections
# in unofficial teams
gov_projects = get_from_cgit('openstack/governance',
'reference/projects.yaml',
{'h': ref})
gov_projects = utils.get_from_cgit('openstack/governance',
'reference/projects.yaml',
{'h': ref})
# The set of retired or removed "legacy" projects from governance
# are merged into the main dict if their retired-on date falls
# later than the after parameter for the qualifying time period
# TODO(fungi): make this a configurable option
old_projects = get_from_cgit('openstack/governance',
'reference/legacy.yaml',
{'h': ref})
old_projects = utils.get_from_cgit('openstack/governance',
'reference/legacy.yaml',
{'h': ref})
for project in old_projects:
for deliverable in old_projects[project]['deliverables']:
retired = old_projects[project]['deliverables'][deliverable].get(
@ -277,7 +209,7 @@ def main(options):
# in governance during transitions and also to filter out repos
# listed in governance which don't actually exist
ger_repos = dict(
[(x.split('/')[-1], x) for x in query_gerrit('projects/')])
[(x.split('/')[-1], x) for x in utils.query_gerrit('projects/')])
# This will be populated with change owners mapped to the
# project-teams maintaining their respective Git repositories
@ -321,7 +253,7 @@ def main(options):
offset = 0
changes = []
while offset >= 0:
changes += query_gerrit('changes/', params={
changes += utils.query_gerrit('changes/', params={
'q': 'project:%s %s' % (ger_repos[repo], match),
'n': '100',
'start': offset,
@ -367,7 +299,7 @@ def main(options):
if new:
# Get the set of all E-mail addresses
# Gerrit knows for this owner's account
emails = query_gerrit(
emails = utils.query_gerrit(
'accounts/%s/emails'
% change['owner']['_account_id'])
@ -552,7 +484,7 @@ def main(options):
'addresses found for account %s' % owner, file=sys.stderr)
continue
for email in [owners[owner]['preferred']] + owners[owner]['extra']:
member = lookup_member(email)
member = utils.lookup_member(email)
if member['data']:
owners[owner]['member'] = member['data'][0]['id']
continue

63
openstack_election/utils.py
View File

@ -20,6 +20,7 @@ import os
import pickle
import pytz
import requests
import sys
import time
import yaml
@ -27,7 +28,6 @@ from six.moves.urllib.parse import quote_plus
from six.moves.urllib.request import urlopen
from openstack_election import config
from openstack_election import owners
# Library constants
@ -52,6 +52,64 @@ def requester(url, params={}, headers={}):
return retry.get(url=url, params=params, headers=headers)
def decode_json(raw):
"""Trap JSON decoding failures and provide more detailed errors"""
# Gerrit's REST API prepends a JSON-breaker to avoid XSS vulnerabilities
if raw.text.startswith(")]}'"):
trimmed = raw.text[4:]
else:
trimmed = raw.text
# Try to decode and bail with much detail if it fails
try:
decoded = json.loads(trimmed)
except Exception:
print('\nrequest returned %s error to query:\n\n %s\n'
'\nwith detail:\n\n %s\n' % (raw, raw.url, trimmed),
file=sys.stderr)
raise
return decoded
def query_gerrit(method, params={}):
"""Query the Gerrit REST API"""
# The base URL to Gerrit REST API
GERRIT_API_URL = 'https://review.openstack.org/'
raw = requester(GERRIT_API_URL + method, params=params,
headers={'Accept': 'application/json'})
return decode_json(raw)
def get_from_cgit(project, obj, params={}):
"""Retrieve a file from the cgit interface"""
url = 'http://git.openstack.org/cgit/' + project + '/plain/' + obj
raw = requester(url, params=params,
headers={'Accept': 'application/json'})
return yaml.safe_load(raw.text)
def lookup_member(email):
"""A requests wrapper to querying the OSF member directory API"""
# The OpenStack foundation member directory lookup API endpoint
MEMBER_LOOKUP_URL = 'https://openstackid-resources.openstack.org/'
# URL pattern for querying foundation members by E-mail address
raw = requester(MEMBER_LOOKUP_URL + '/api/public/v1/members',
params={'filter[]': [
'group_slug==foundation-members',
'email==' + email,
]},
headers={'Accept': 'application/json'},
)
return decode_json(raw)
def load_exceptions():
global exceptions
exceptions = {}
@ -73,6 +131,7 @@ def gerrit_datetime(dt):
return dt.strftime('%Y-%m-%d %H:%M:%S %z')
# TODO(tonyb): this is now basically a duplicate of query_gerrit()
def gerrit_query(url, params=None):
r = requester(url, params=params)
if r.status_code == 200:
@ -226,7 +285,7 @@ def build_candidates_list(election=conf['release']):
for candidate_file in file_list:
filepath = os.path.join(project_prefix, candidate_file)
email = get_email(filepath)
member = owners.lookup_member(email)
member = lookup_member(email)
candidates_list.append(
{
'url': ('%s/%s/plain/%s' %