Include a -v/--verbose option for check commands

So as to increase transparency of the candidate verification checks,
add a --verbose option to all of the various candidate checking
command scripts. Supplying it once returns check result URLs for
clarity, while adding it twice also displays the query URLs used.

Set double verbosity for the check review tox testenv, to aid
election officials in reviewing nominees for valid candidacy.

Also update the Gerrit and Git URLs for the OpenDev transition and
correct a couple of docstrings for utility functions.

Change-Id: I5f6fa4e2c2c6058ba5090078bbdf9dd9f31f692e

openstack_election/check_candidacy.py

@@ -23,7 +23,7 @@ from openstack_election import utils
 # FIXME: Printing from library function isn't great.
 #        change API to return the messages and let the consumer decide what to
 #        do with them
-def check_candidate(project_name, email, projects, limit=1):
+def check_candidate(project_name, email, projects, limit=1, verbose=0):
     def pretty_datetime(dt_str):
         dt = datetime.datetime.strptime(dt_str.split('.')[0],
                                         '%Y-%m-%d %H:%M:%S')
@@ -57,15 +57,14 @@ def check_candidate(project_name, email, projects, limit=1):
                           owner, repo_name))
                 if branch:
                     query += (' branch:%s' % (branch))
-                print('Checking %s for merged changes by %s' %
+                if verbose >= 1:
-                      (repo_name, email))
+                    print('Checking %s for merged changes by %s' %
-                for review in utils.get_reviews(query):
+                          (repo_name, email))
-                    url = ('%s/%s/commit/?id=%s' % (
+                for review in utils.get_reviews(query, verbose=verbose):
-                           utils.CGIT_URL, review['project'],
+                    print('Found: %s/%s merged on %s to %s for %s' % (
-                           review['current_revision']))
+                        utils.GERRIT_BASE, review['_number'],
-                    print('%2d: %s %s' %
+                        pretty_datetime(review['submitted']), repo_name,
-                          (found, pretty_datetime(review['submitted']),
+                        project_name))
-                           url))
                     found += 1
                     if found >= limit:
                         return found

openstack_election/cmds/check_all_candidacies.py

@@ -24,15 +24,15 @@ from six.moves import input
 results = []
 
 
-def get_reviews():
+def get_reviews(verbose=0):
     return utils.get_reviews('is:open project:%s file:^%s/%s/.*' %
                              (utils.ELECTION_REPO, utils.CANDIDATE_PATH,
-                              utils.conf['release']))
+                              utils.conf['release']), verbose=verbose)
 
 
-def print_member(filepath):
+def print_member(filepath, verbose=0):
     email = utils.get_email(filepath)
-    member = utils.lookup_member(email)
+    member = utils.lookup_member(email, verbose=verbose)
     member_id = member.get('data', [{}])[0].get('id')
     base = 'https://www.openstack.org/community/members/profile'
     print('OSF member profile: %s/%s' % (base, member_id))
@@ -52,12 +52,14 @@ def main():
                         action='store_true',
                         help=('Pause after each review to manually post '
                               'results'))
+    parser.add_argument('-v', '--verbose', action="count", default=0,
+                        help='Increase program verbosity')
 
     args = parser.parse_args()
     projects = utils.get_projects(tag=args.tag, fallback_to_master=True)
     election_type = utils.conf.get('election_type', '').lower()
 
-    for review in get_reviews():
+    for review in get_reviews(verbose=args.verbose):
         if review['status'] != 'NEW':
             continue
 
@@ -75,7 +77,8 @@ def main():
 
             candiate_ok = checks.validate_filename(filepath)
             if candiate_ok:
-                candiate_ok = checks.validate_member(filepath)
+                candiate_ok = checks.validate_member(filepath,
+                                                     verbose=args.verbose)
 
             if candiate_ok:
                 # If we're a PTL election OR if the team is not TC we need
@@ -85,9 +88,10 @@ def main():
                     if args.interactive:
                         print('The following commit and profile validate this '
                               'candidate:')
-                    candiate_ok = checks.check_for_changes(projects, filepath,
+                    candiate_ok = checks.check_for_changes(
-                                                           args.limit)
+                        projects, filepath, args.limit,
-                    print_member(filepath)
+                        verbose=args.verbose)
+                    print_member(filepath, verbose=args.verbose)
                 else:
                     print('Not checking for changes as this is a TC election')
             else:

openstack_election/cmds/check_candidacy.py

@@ -32,9 +32,11 @@ def main():
     parser.add_argument('--tag', dest='tag', default=utils.conf['tag'],
                         help=('The governance tag to validate against.  '
                               'Default: %(default)s'))
+    parser.add_argument('-v', '--verbose', action="count", default=0,
+                        help='Increase program verbosity')
 
     args = parser.parse_args()
-    review = utils.get_reviews(args.change_id)[0]
+    review = utils.get_reviews(args.change_id, verbose=args.verbose)[0]
     owner = review.get('owner', {})
     if args.limit < 0:
         args.limit = 100

openstack_election/cmds/check_manual.py

@@ -35,6 +35,8 @@ def main():
     parser.add_argument('--tag', dest='tag', default=utils.conf['tag'],
                         help=('The governance tag to validate against.  '
                               'Default: %(default)s'))
+    parser.add_argument('-v', '--verbose', action="count", default=0,
+                        help='Increase program verbosity')
 
     args = parser.parse_args()
     if args.limit < 0:
@@ -48,7 +50,8 @@ def main():
         return 1
 
     if check_candidacy.check_candidate(args.project_name, args.email,
-                                       projects, limit=args.limit):
+                                       projects, limit=args.limit,
+                                       verbose=args.verbose):
         print('SUCCESS: %s is a valid candidate\n\n' % (args.email))
         return 0
     else:

openstack_election/cmds/ci_check_all_candidate_files.py

@@ -48,12 +48,12 @@ def validate_filename(filepath):
     return is_valid
 
 
-def validate_member(filepath):
+def validate_member(filepath, verbose=0):
     print('Validate email address is OSF member')
     print('------------------------------------')
 
     email = utils.get_email(filepath)
-    member = utils.lookup_member(email)
+    member = utils.lookup_member(email, verbose=verbose)
     is_valid = member.get('data', []) != []
 
     print('Email address: %s %s' % (email,
@@ -62,7 +62,7 @@ def validate_member(filepath):
     return is_valid
 
 
-def check_for_changes(projects, filepath, limit):
+def check_for_changes(projects, filepath, limit, verbose=0):
     print('Looking for validating changes')
     print('------------------------------')
 
@@ -71,7 +71,10 @@ def check_for_changes(projects, filepath, limit):
     project_name = utils.dir2name(project_name, projects)
 
     changes_found = check_candidacy.check_candidate(project_name, email,
-                                                    projects, limit)
+                                                    projects, limit,
+                                                    verbose=verbose)
+    print('Email address: %s %s' % (
+        email, {True: 'PASS', False: 'FAIL'}[changes_found]))
     print('')
     return bool(changes_found)
 
@@ -113,6 +116,8 @@ def main():
     parser.add_argument('files',
                         nargs='*',
                         help='Candidate files to validate.')
+    parser.add_argument('-v', '--verbose', action="count", default=0,
+                        help='Increase program verbosity')
 
     args = parser.parse_args()
     errors = False
@@ -146,13 +151,14 @@ def main():
         candidate_ok = True
 
         candidate_ok &= validate_filename(filepath)
-        candidate_ok &= validate_member(filepath)
+        candidate_ok &= validate_member(filepath, verbose=args.verbose)
 
         if candidate_ok:
             if (election_type == 'ptl'
                     or (election_type == 'combined' and team != 'TC')):
                 candidate_ok &= check_for_changes(projects, filepath,
-                                                  args.limit)
+                                                  args.limit,
+                                                  verbose=args.verbose)
 
         errors |= not candidate_ok
 

openstack_election/cmds/template_emails.py

@@ -12,7 +12,7 @@ from openstack_election import utils
 
 conf = config.load_conf()
 
-REFERENCE_URL = '%s?id=%s' % (utils.PROJECTS_URL, conf['tag'])
+REFERENCE_URL = utils.PROJECTS_URL % '/'.join(('tag', conf['tag']))
 LEADERLESS_URL = ('https://governance.openstack.org/resolutions/'
                   '20141128-elections-process-for-leaderless-programs.html')
 

openstack_election/owners.py

@@ -162,7 +162,7 @@ def main(options):
     elif 'ref' in config:
         ref = config['ref']
     else:
-        ref = 'refs/heads/master'
+        ref = 'branch/master'
 
     # Gerrit change query additions
     if options.sieve:
@@ -184,9 +184,8 @@ def main(options):
     if projects_file:
         gov_projects = utils.load_yaml(open(projects_file).read())
     else:
-        gov_projects = utils.get_from_cgit('openstack/governance',
+        gov_projects = utils.get_from_git('openstack/governance',
-                                           'reference/projects.yaml',
+                                          '%s/reference/projects.yaml' % ref)
-                                           {'h': ref})
 
     # The set of retired or removed "legacy" projects from governance
     # are merged into the main dict if their retired-on date falls
@@ -196,9 +195,8 @@ def main(options):
     elif projects_file:
         old_projects = []
     else:
-        old_projects = utils.get_from_cgit('openstack/governance',
+        old_projects = utils.get_from_git('openstack/governance',
-                                           'reference/legacy.yaml',
+                                          '%s/reference/legacy.yaml' % ref)
-                                           {'h': ref})
     for project in old_projects:
         for deliverable in old_projects[project]['deliverables']:
             retired = old_projects[project]['deliverables'][deliverable].get(

openstack_election/templates/tc_campaigning_kickoff.j2

@@ -21,4 +21,4 @@ concern, and the electorate has the best information to determine the ideal
 TC composition to address these and other issues that may arise.
 
 [1] https://governance.openstack.org/election/
-[2] https://git.openstack.org/cgit/openstack/election/tree/candidates/{{ release }}/TC
+[2] https://opendev.org/openstack/election/src/branch/master/candidates/{{ release }}/TC

openstack_election/utils.py

@@ -34,24 +34,26 @@ from openstack_election import exception
 
 # Library constants
 CANDIDATE_PATH = 'candidates'
-GERRIT_BASE = 'https://review.openstack.org'
+GERRIT_BASE = 'https://review.opendev.org'
 ELECTION_REPO = 'openstack/election'
-CGIT_URL = 'https://git.openstack.org/cgit'
+GIT_URL = 'https://opendev.org/'
-PROJECTS_URL = ('%s/openstack/governance/plain/reference/projects.yaml' %
+PROJECTS_URL = GIT_URL + 'openstack/governance/raw/%s/reference/projects.yaml'
-                (CGIT_URL))
 
 conf = config.load_conf()
 exceptions = None
 
 
 # Generic functions
-def requester(url, params={}, headers={}):
+def requester(url, params={}, headers={}, verbose=0):
     """A requests wrapper to consistently retry HTTPS queries"""
 
     # Try up to 3 times
     retry = requests.Session()
     retry.mount("https://", requests.adapters.HTTPAdapter(max_retries=3))
-    return retry.get(url=url, params=params, headers=headers)
+    raw = retry.get(url=url, params=params, headers=headers)
+    if verbose >= 2:
+        print("Queried: %s" % raw.url)
+    return raw
 
 
 def decode_json(raw):
@@ -74,43 +76,41 @@ def decode_json(raw):
     return decoded
 
 
-def query_gerrit(method, params={}):
+def query_gerrit(method, params={}, verbose=0):
     """Query the Gerrit REST API"""
 
-    # The base URL to Gerrit REST API
+    return decode_json(requester("%s/%s" % (GERRIT_BASE, method),
-    GERRIT_API_URL = 'https://review.openstack.org/'
+                                 params=params,
-
+                                 headers={'Accept': 'application/json'},
-    raw = requester(GERRIT_API_URL + method, params=params,
+                                 verbose=verbose))
-                    headers={'Accept': 'application/json'})
-    return decode_json(raw)
 
 
 def load_yaml(yaml_stream):
-    """Retrieve a file from the cgit interface"""
+    """Wrapper to load and return YAML data"""
 
     return yaml.safe_load(yaml_stream)
 
 
-def get_from_cgit(project, obj, params={}):
+def get_from_git(project, obj, params={}, verbose=0):
-    """Retrieve a file from the cgit interface"""
+    """Retrieve a file from the Gitea interface"""
 
-    url = 'https://git.openstack.org/cgit/' + project + '/plain/' + obj
+    url = "%s%s/raw/%s" % (GIT_URL, project, obj)
-    raw = requester(url, params=params,
+    return load_yaml(requester(url, params=params,
-                    headers={'Accept': 'application/json'})
+                               headers={'Accept': 'application/json'},
-    return load_yaml(raw.text)
+                               verbose=verbose).text)
 
 
 def get_series_data():
-    return get_from_cgit('openstack/releases',
+    return get_from_git('openstack/releases',
-                         'deliverables/series_status.yaml')
+                        'branch/master/deliverables/series_status.yaml')
 
 
 def get_schedule_data(series):
-    return get_from_cgit('openstack/releases',
+    return get_from_git('openstack/releases',
-                         'doc/source/%s/schedule.yaml' % (series))
+                        'branch/master/doc/source/%s/schedule.yaml' % (series))
 
 
-def lookup_member(email):
+def lookup_member(email, verbose=0):
     """A requests wrapper to querying the OSF member directory API"""
 
     # The OpenStack foundation member directory lookup API endpoint
@@ -123,9 +123,17 @@ def lookup_member(email):
                         'email==' + email,
                         ]},
                     headers={'Accept': 'application/json'},
+                    verbose=verbose,
                     )
+    result = decode_json(raw)
 
-    return decode_json(raw)
+    # Print the profile if verbosity is 1 or higher
+    if verbose >= 1 and result['data']:
+        print("Found: "
+              "https://openstack.org/community/members/profile/%s"
+              % result['data'][0]['id'])
+
+    return result
 
 
 def load_exceptions():
@@ -150,12 +158,13 @@ def gerrit_datetime(dt):
 
 
 # TODO(tonyb): this is now basically a duplicate of query_gerrit()
-def gerrit_query(url, params=None):
+def gerrit_query(url, params=None, verbose=0):
-    r = requester(url, params=params)
+    r = requester(url, params=params, verbose=verbose)
     if r.status_code == 200:
         data = json.loads(r.text[4:])
     else:
         data = []
+
     return data
 
 
@@ -199,12 +208,12 @@ def get_fullname(member, filepath=None):
     return full_name
 
 
-def get_reviews(query):
+def get_reviews(query, verbose=0):
     opts = ['CURRENT_REVISION', 'CURRENT_FILES', 'DETAILED_ACCOUNTS']
     opts_str = '&o=%s' % ('&o='.join(opts))
     url = ('%s/changes/?q=%s%s' %
            (GERRIT_BASE, quote_plus(query, safe='/:=><^.*'), opts_str))
-    return gerrit_query(url)
+    return gerrit_query(url, verbose=verbose)
 
 
 def candidate_files(review):
@@ -222,12 +231,12 @@ def check_atc_date(atc):
 
 
 def _get_projects(tag=None):
-    url = PROJECTS_URL
-    cache_file = '.projects.pkl'
-
     if tag:
-        url += '?h=%s' % tag
+        url = PROJECTS_URL % '/'.join(('tag', tag))
         cache_file = '.projects.%s.pkl' % tag
+    else:
+        url = PROJECTS_URL % 'branch/master'
+        cache_file = '.projects.pkl'
 
     # Refresh the cache if it's not there or if it's older than a week
     if (not os.path.isfile(cache_file) or
@@ -333,8 +342,8 @@ def build_candidates_list(election=conf['release']):
             raise exception.MemberNotFoundException(email=email)
 
         candidates_lists[project].append({
-            'url': ('%s/%s/plain/%s' %
+            'url': ('%s%s/raw/branch/master/%s' %
-                    (CGIT_URL, ELECTION_REPO,
+                    (GIT_URL, ELECTION_REPO,
                      quote_plus(filepath, safe='/'))),
             'email': email,
             'ircname': get_irc(member),

tox.ini

@@ -27,7 +27,7 @@ commands = {posargs}
 commands = sphinx-build -v -W -b html -d doc/build/doctrees doc/source doc/build/html
 
 [testenv:ci-checks-review]
-commands = ci-check-all-candidate-files {posargs:--HEAD}
+commands = ci-check-all-candidate-files -v -v {posargs:--HEAD}
 
 [testenv:ci-checks-election]
 commands = ci-check-all-candidate-files