openstack/election
Code Issues Proposed changes

Include a -v/--verbose option for check commands

Browse Source 
So as to increase transparency of the candidate verification checks,
add a --verbose option to all of the various candidate checking
command scripts. Supplying it once returns check result URLs for
clarity, while adding it twice also displays the query URLs used.

Set double verbosity for the check review tox testenv, to aid
election officials in reviewing nominees for valid candidacy.

Also update the Gerrit and Git URLs for the OpenDev transition and
correct a couple of docstrings for utility functions.

Change-Id: I5f6fa4e2c2c6058ba5090078bbdf9dd9f31f692e
This commit is contained in:
Jeremy Stanley 2019-08-29 17:05:09 +00:00
parent a3b4725f1b
commit c24fab9d73
10 changed files with 94 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
openstack_election/check_candidacy.py
View File

@ -23,7 +23,7 @@ from openstack_election import utils
# FIXME: Printing from library function isn't great.
# change API to return the messages and let the consumer decide what to
# do with them
def check_candidate(project_name, email, projects, limit=1):
def check_candidate(project_name, email, projects, limit=1, verbose=0):
def pretty_datetime(dt_str):
dt = datetime.datetime.strptime(dt_str.split('.')[0],
'%Y-%m-%d %H:%M:%S')
@ -57,15 +57,14 @@ def check_candidate(project_name, email, projects, limit=1):
owner, repo_name))
if branch:
query += (' branch:%s' % (branch))
print('Checking %s for merged changes by %s' %
(repo_name, email))
for review in utils.get_reviews(query):
url = ('%s/%s/commit/?id=%s' % (
utils.CGIT_URL, review['project'],
review['current_revision']))
print('%2d: %s %s' %
(found, pretty_datetime(review['submitted']),
url))
if verbose >= 1:
print('Checking %s for merged changes by %s' %
(repo_name, email))
for review in utils.get_reviews(query, verbose=verbose):
print('Found: %s/%s merged on %s to %s for %s' % (
utils.GERRIT_BASE, review['_number'],
pretty_datetime(review['submitted']), repo_name,
project_name))
found += 1
if found >= limit:
return found

22
openstack_election/cmds/check_all_candidacies.py
View File

@ -24,15 +24,15 @@ from six.moves import input
results = []
def get_reviews():
def get_reviews(verbose=0):
return utils.get_reviews('is:open project:%s file:^%s/%s/.*' %
(utils.ELECTION_REPO, utils.CANDIDATE_PATH,
utils.conf['release']))
utils.conf['release']), verbose=verbose)
def print_member(filepath):
def print_member(filepath, verbose=0):
email = utils.get_email(filepath)
member = utils.lookup_member(email)
member = utils.lookup_member(email, verbose=verbose)
member_id = member.get('data', [{}])[0].get('id')
base = 'https://www.openstack.org/community/members/profile'
print('OSF member profile: %s/%s' % (base, member_id))
@ -52,12 +52,14 @@ def main():
action='store_true',
help=('Pause after each review to manually post '
'results'))
parser.add_argument('-v', '--verbose', action="count", default=0,
help='Increase program verbosity')
args = parser.parse_args()
projects = utils.get_projects(tag=args.tag, fallback_to_master=True)
election_type = utils.conf.get('election_type', '').lower()
for review in get_reviews():
for review in get_reviews(verbose=args.verbose):
if review['status'] != 'NEW':
continue
@ -75,7 +77,8 @@ def main():
candiate_ok = checks.validate_filename(filepath)
if candiate_ok:
candiate_ok = checks.validate_member(filepath)
candiate_ok = checks.validate_member(filepath,
verbose=args.verbose)
if candiate_ok:
# If we're a PTL election OR if the team is not TC we need
@ -85,9 +88,10 @@ def main():
if args.interactive:
print('The following commit and profile validate this '
'candidate:')
candiate_ok = checks.check_for_changes(projects, filepath,
args.limit)
print_member(filepath)
candiate_ok = checks.check_for_changes(
projects, filepath, args.limit,
verbose=args.verbose)
print_member(filepath, verbose=args.verbose)
else:
print('Not checking for changes as this is a TC election')
else:

4
openstack_election/cmds/check_candidacy.py
View File

@ -32,9 +32,11 @@ def main():
parser.add_argument('--tag', dest='tag', default=utils.conf['tag'],
help=('The governance tag to validate against. '
'Default: %(default)s'))
parser.add_argument('-v', '--verbose', action="count", default=0,
help='Increase program verbosity')
args = parser.parse_args()
review = utils.get_reviews(args.change_id)[0]
review = utils.get_reviews(args.change_id, verbose=args.verbose)[0]
owner = review.get('owner', {})
if args.limit < 0:
args.limit = 100

5
openstack_election/cmds/check_manual.py
View File

@ -35,6 +35,8 @@ def main():
parser.add_argument('--tag', dest='tag', default=utils.conf['tag'],
help=('The governance tag to validate against. '
'Default: %(default)s'))
parser.add_argument('-v', '--verbose', action="count", default=0,
help='Increase program verbosity')
args = parser.parse_args()
if args.limit < 0:
@ -48,7 +50,8 @@ def main():
return 1
if check_candidacy.check_candidate(args.project_name, args.email,
projects, limit=args.limit):
projects, limit=args.limit,
verbose=args.verbose):
print('SUCCESS: %s is a valid candidate\n\n' % (args.email))
return 0
else:

18
openstack_election/cmds/ci_check_all_candidate_files.py
View File

@ -48,12 +48,12 @@ def validate_filename(filepath):
return is_valid
def validate_member(filepath):
def validate_member(filepath, verbose=0):
print('Validate email address is OSF member')
print('------------------------------------')
email = utils.get_email(filepath)
member = utils.lookup_member(email)
member = utils.lookup_member(email, verbose=verbose)
is_valid = member.get('data', []) != []
print('Email address: %s %s' % (email,
@ -62,7 +62,7 @@ def validate_member(filepath):
return is_valid
def check_for_changes(projects, filepath, limit):
def check_for_changes(projects, filepath, limit, verbose=0):
print('Looking for validating changes')
print('------------------------------')
@ -71,7 +71,10 @@ def check_for_changes(projects, filepath, limit):
project_name = utils.dir2name(project_name, projects)
changes_found = check_candidacy.check_candidate(project_name, email,
projects, limit)
projects, limit,
verbose=verbose)
print('Email address: %s %s' % (
email, {True: 'PASS', False: 'FAIL'}[changes_found]))
print('')
return bool(changes_found)
@ -113,6 +116,8 @@ def main():
parser.add_argument('files',
nargs='*',
help='Candidate files to validate.')
parser.add_argument('-v', '--verbose', action="count", default=0,
help='Increase program verbosity')
args = parser.parse_args()
errors = False
@ -146,13 +151,14 @@ def main():
candidate_ok = True
candidate_ok &= validate_filename(filepath)
candidate_ok &= validate_member(filepath)
candidate_ok &= validate_member(filepath, verbose=args.verbose)
if candidate_ok:
if (election_type == 'ptl'
or (election_type == 'combined' and team != 'TC')):
candidate_ok &= check_for_changes(projects, filepath,
args.limit)
args.limit,
verbose=args.verbose)
errors |= not candidate_ok

2
openstack_election/cmds/template_emails.py
View File

@ -12,7 +12,7 @@ from openstack_election import utils
conf = config.load_conf()
REFERENCE_URL = '%s?id=%s' % (utils.PROJECTS_URL, conf['tag'])
REFERENCE_URL = utils.PROJECTS_URL % '/'.join(('tag', conf['tag']))
LEADERLESS_URL = ('https://governance.openstack.org/resolutions/'
'20141128-elections-process-for-leaderless-programs.html')

12
openstack_election/owners.py
View File

@ -162,7 +162,7 @@ def main(options):
elif 'ref' in config:
ref = config['ref']
else:
ref = 'refs/heads/master'
ref = 'branch/master'
# Gerrit change query additions
if options.sieve:
@ -184,9 +184,8 @@ def main(options):
if projects_file:
gov_projects = utils.load_yaml(open(projects_file).read())
else:
gov_projects = utils.get_from_cgit('openstack/governance',
'reference/projects.yaml',
{'h': ref})
gov_projects = utils.get_from_git('openstack/governance',
'%s/reference/projects.yaml' % ref)
# The set of retired or removed "legacy" projects from governance
# are merged into the main dict if their retired-on date falls
@ -196,9 +195,8 @@ def main(options):
elif projects_file:
old_projects = []
else:
old_projects = utils.get_from_cgit('openstack/governance',
'reference/legacy.yaml',
{'h': ref})
old_projects = utils.get_from_git('openstack/governance',
'%s/reference/legacy.yaml' % ref)
for project in old_projects:
for deliverable in old_projects[project]['deliverables']:
retired = old_projects[project]['deliverables'][deliverable].get(

2
openstack_election/templates/tc_campaigning_kickoff.j2
View File

@ -21,4 +21,4 @@ concern, and the electorate has the best information to determine the ideal
TC composition to address these and other issues that may arise.
[1] https://governance.openstack.org/election/
[2] https://git.openstack.org/cgit/openstack/election/tree/candidates/{{ release }}/TC
[2] https://opendev.org/openstack/election/src/branch/master/candidates/{{ release }}/TC

81
openstack_election/utils.py
View File

@ -34,24 +34,26 @@ from openstack_election import exception
# Library constants
CANDIDATE_PATH = 'candidates'
GERRIT_BASE = 'https://review.openstack.org'
GERRIT_BASE = 'https://review.opendev.org'
ELECTION_REPO = 'openstack/election'
CGIT_URL = 'https://git.openstack.org/cgit'
PROJECTS_URL = ('%s/openstack/governance/plain/reference/projects.yaml' %
(CGIT_URL))
GIT_URL = 'https://opendev.org/'
PROJECTS_URL = GIT_URL + 'openstack/governance/raw/%s/reference/projects.yaml'
conf = config.load_conf()
exceptions = None
# Generic functions
def requester(url, params={}, headers={}):
def requester(url, params={}, headers={}, verbose=0):
"""A requests wrapper to consistently retry HTTPS queries"""
# Try up to 3 times
retry = requests.Session()
retry.mount("https://", requests.adapters.HTTPAdapter(max_retries=3))
return retry.get(url=url, params=params, headers=headers)
raw = retry.get(url=url, params=params, headers=headers)
if verbose >= 2:
print("Queried: %s" % raw.url)
return raw
def decode_json(raw):
@ -74,43 +76,41 @@ def decode_json(raw):
return decoded
def query_gerrit(method, params={}):
def query_gerrit(method, params={}, verbose=0):
"""Query the Gerrit REST API"""
# The base URL to Gerrit REST API
GERRIT_API_URL = 'https://review.openstack.org/'
raw = requester(GERRIT_API_URL + method, params=params,
headers={'Accept': 'application/json'})
return decode_json(raw)
return decode_json(requester("%s/%s" % (GERRIT_BASE, method),
params=params,
headers={'Accept': 'application/json'},
verbose=verbose))
def load_yaml(yaml_stream):
"""Retrieve a file from the cgit interface"""
"""Wrapper to load and return YAML data"""
return yaml.safe_load(yaml_stream)
def get_from_cgit(project, obj, params={}):
"""Retrieve a file from the cgit interface"""
def get_from_git(project, obj, params={}, verbose=0):
"""Retrieve a file from the Gitea interface"""
url = 'https://git.openstack.org/cgit/' + project + '/plain/' + obj
raw = requester(url, params=params,
headers={'Accept': 'application/json'})
return load_yaml(raw.text)
url = "%s%s/raw/%s" % (GIT_URL, project, obj)
return load_yaml(requester(url, params=params,
headers={'Accept': 'application/json'},
verbose=verbose).text)
def get_series_data():
return get_from_cgit('openstack/releases',
'deliverables/series_status.yaml')
return get_from_git('openstack/releases',
'branch/master/deliverables/series_status.yaml')
def get_schedule_data(series):
return get_from_cgit('openstack/releases',
'doc/source/%s/schedule.yaml' % (series))
return get_from_git('openstack/releases',
'branch/master/doc/source/%s/schedule.yaml' % (series))
def lookup_member(email):
def lookup_member(email, verbose=0):
"""A requests wrapper to querying the OSF member directory API"""
# The OpenStack foundation member directory lookup API endpoint
@ -123,9 +123,17 @@ def lookup_member(email):
'email==' + email,
]},
headers={'Accept': 'application/json'},
verbose=verbose,
)
result = decode_json(raw)
return decode_json(raw)
# Print the profile if verbosity is 1 or higher
if verbose >= 1 and result['data']:
print("Found: "
"https://openstack.org/community/members/profile/%s"
% result['data'][0]['id'])
return result
def load_exceptions():
@ -150,12 +158,13 @@ def gerrit_datetime(dt):
# TODO(tonyb): this is now basically a duplicate of query_gerrit()
def gerrit_query(url, params=None):
r = requester(url, params=params)
def gerrit_query(url, params=None, verbose=0):
r = requester(url, params=params, verbose=verbose)
if r.status_code == 200:
data = json.loads(r.text[4:])
else:
data = []
return data
@ -199,12 +208,12 @@ def get_fullname(member, filepath=None):
return full_name
def get_reviews(query):
def get_reviews(query, verbose=0):
opts = ['CURRENT_REVISION', 'CURRENT_FILES', 'DETAILED_ACCOUNTS']
opts_str = '&o=%s' % ('&o='.join(opts))
url = ('%s/changes/?q=%s%s' %
(GERRIT_BASE, quote_plus(query, safe='/:=><^.*'), opts_str))
return gerrit_query(url)
return gerrit_query(url, verbose=verbose)
def candidate_files(review):
@ -222,12 +231,12 @@ def check_atc_date(atc):
def _get_projects(tag=None):
url = PROJECTS_URL
cache_file = '.projects.pkl'
if tag:
url += '?h=%s' % tag
url = PROJECTS_URL % '/'.join(('tag', tag))
cache_file = '.projects.%s.pkl' % tag
else:
url = PROJECTS_URL % 'branch/master'
cache_file = '.projects.pkl'
# Refresh the cache if it's not there or if it's older than a week
if (not os.path.isfile(cache_file) or
@ -333,8 +342,8 @@ def build_candidates_list(election=conf['release']):
raise exception.MemberNotFoundException(email=email)
candidates_lists[project].append({
'url': ('%s/%s/plain/%s' %
(CGIT_URL, ELECTION_REPO,
'url': ('%s%s/raw/branch/master/%s' %
(GIT_URL, ELECTION_REPO,
quote_plus(filepath, safe='/'))),
'email': email,
'ircname': get_irc(member),

2
tox.ini
View File

@ -27,7 +27,7 @@ commands = {posargs}
commands = sphinx-build -v -W -b html -d doc/build/doctrees doc/source doc/build/html
[testenv:ci-checks-review]
commands = ci-check-all-candidate-files {posargs:--HEAD}
commands = ci-check-all-candidate-files -v -v {posargs:--HEAD}
[testenv:ci-checks-election]
commands = ci-check-all-candidate-files