diff --git a/openstack_election/cmds/generate_rolls.py b/openstack_election/cmds/generate_rolls.py new file mode 100755 index 00000000..35a3382d --- /dev/null +++ b/openstack_election/cmds/generate_rolls.py @@ -0,0 +1,113 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +import glob +import hashlib +import os +import shutil +import sys +import tempfile +import time + +from six.moves.urllib.request import urlopen + +from openstack_election import utils + +CGIT_URL = "http://git.openstack.org/cgit/openstack-infra/system-config/" \ + "plain/tools/owners.py" + +# Exclude the system / bot accounts +# OpenStack Release Bot: +# curl https://review.openstack.org/accounts/22816 +# OpenStack Proposal Bot +# curl https://review.openstack.org/accounts/11131 +MINUS_BOT_ACCOUNTS = ["-i", "11131", "-i", "22816"] + + +def main(): + start = utils.conf['timeframe']['start'] + end = utils.conf['timeframe']['end'] + description = ('Generate electorate rolls') + parser = argparse.ArgumentParser(description) + parser.add_argument('--tag', dest='tag', default=utils.conf['tag'], + help=('The governance tag to validate against. ' + 'Default: %(default)s')) + parser.add_argument('--rolls_dir', default="./rolls/%s" % ( + utils.conf['tag']), help=('The output directory. ' + 'Default: %(default)s')) + parser.add_argument('--before', default=end.strftime("%Y-%m-%d %H-%M-%S"), + help=('The start date. Default: %(default)s')) + parser.add_argument('--after', default=start.strftime("%Y-%m-%d"), + help=('The end date. Default: %(default)s')) + parser.add_argument('--without-stable', dest='with_stable', default=True, + action='store_false', help=('Do not include stables')) + parser.add_argument('--cached-owners-script', default=False, + action='store_true', help=('Do not fetch the script')) + + args = parser.parse_args() + + print("Run information:") + print(" - TAG : %s" % args.tag) + print(" - ROLLS : %s" % args.rolls_dir) + print(" - STABLE : %s" % args.with_stable) + print(" - BEFORE : %s" % args.before) + print(" - AFTER : %s" % args.after) + + if os.path.isdir(args.rolls_dir): + shutil.rmtree(args.rolls_dir) + os.makedirs(args.rolls_dir, 0o700) + + os.chdir(os.path.dirname(args.rolls_dir)) + if not args.cached_owners_script: + print("Grabbing script from: %s" % CGIT_URL) + with open("owners.py", "wb") as owner_file: + owner_file.write(urlopen(CGIT_URL).read()) + + sys.path.append(os.getcwd()) + import owners + print("Starting roll generation @%s" % time.ctime()) + owners.main(["owners.py", "-a", args.after, "-b", args.before, + "-o", args.tag, "-r", args.tag] + MINUS_BOT_ACCOUNTS) + print("Finished roll generation @%s" % time.ctime()) + + if args.with_stable: + tmp_dir = tempfile.mkdtemp(prefix='election.') + print("Starting (Stable) roll generation @%s" % time.ctime()) + owners.main(["owners.py", "-a", args.after, "-b", args.before, + "-o", tmp_dir, "-r", args.tag, "-n", + "-s", "branch:^stable/.*"] + MINUS_BOT_ACCOUNTS) + print("Finished (Stable) roll generation @%s" % time.ctime()) + shutil.copy("%s/_electorate.txt" % tmp_dir, + "./%s/stable_branch_maintenance.txt" % args.tag) + shutil.copy("%s/_all_owners.yaml" % tmp_dir, + "./%s/stable_branch_maintenance.yaml" % args.tag) + shutil.rmtree(tmp_dir) + + print("Electoral rolls SHA256sum") + with open("%s.shasums" % args.tag, "wb") as ofile: + for fname in sorted(glob.glob("./%s/*.txt" % args.tag)): + # Sort the list + with open(fname) as list_file: + roll = sorted(list_file.readlines()) + content = "".join(roll).encode('utf-8') + with open(fname, "wb") as list_file: + list_file.write(content) + # Generate the hash + h = hashlib.sha256(content).hexdigest() + s = "%s %s" % (h, fname) + print(s) + ofile.write(("%s\n" % s).encode('utf-8')) diff --git a/setup.cfg b/setup.cfg index 4d66a2e4..30f0e5d5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,6 +28,7 @@ console_scripts = close-election = openstack_election.cmds.close_election:main search-rolls = openstack_election.cmds.search_rolls:main create-directories = openstack_election.cmds.create_directories:main + generate-rolls = openstack_election.cmds.generate_rolls:main [build_sphinx] all_files = 1 diff --git a/tools/generate-rolls.sh b/tools/generate-rolls.sh deleted file mode 100644 index 2db08300..00000000 --- a/tools/generate-rolls.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env bash - -CGIT_URL=http://git.openstack.org/cgit/openstack-infra/system-config/plain/tools/owners.py -OWNERS=$(basename "$CGIT_URL") -WITH_STABLE=1 - -while [ $# -gt 0 ] ; do - case "$1" in - --tag) - TAG=$2 - shift 1 - ;; - --before) - BEFORE=$2 - shift 1 - ;; - --after) - AFTER=$2 - shift 1 - ;; - --without-stable) - WITH_STABLE=0 - ;; - *) - ;; - esac - shift 1 -done - -if [ -z "$TAG" ] ; then - echo $(basename "$0"): You must supply a --tag >&2 - exit 1 -fi - -if [ -z "$AFTER" ] ; then - echo $(basename "$0"): You must supply a --after >&2 - exit 1 -fi - -if [ -z "$BEFORE" ] ; then - echo $(basename "$0"): You must supply a --before >&2 - exit 1 -fi - - -ROLLS_DIR="./rolls/${TAG}" - -echo "Run information:" -echo " - TAG :" $TAG -echo " - ROLLS :" $ROLLS_DIR -echo " - STABLE :" $WITH_STABLE -echo " - AFTER :" $AFTER -echo " - BEFORE :" $BEFORE - -rm -rf "${ROLLS_DIR}" -mkdir -p "${ROLLS_DIR}" - -cd $(dirname "${ROLLS_DIR}") - -# Building the venv isn't really needed everytime -if [ ! -d venv ] ; then - echo "Building tools venv" - ( - # owners.py isn't python3 safe as string.maketrans is now - # str.maketrans - virtualenv -p python2 venv - ./venv/bin/pip install -U pip setuptools wheel - ./venv/bin/pip install pyyaml requests ndg-httpsclient - ) >/dev/null 2>&1 - if [ $? -ne 0 ] ; then - echo Creating Virtualenv failed >&2 - exit 1 - fi -else - echo "Using existing venv" -fi - -# Always get the newest script in case a bug has been fixed. -rm -f "./${OWNERS}" >/dev/null 2>&1 -echo "Grabbing script from: ${CGIT_URL}" -wget "${CGIT_URL}" -O "./${OWNERS}" -o /dev/null -if [ ! -e "./${OWNERS}" ] ; then - echo Failed to download ${CGIT_URL} >&2 - exit 1 -fi - -echo "Starting roll generation @" $(TZ=UTC date) -# Exclude the system / bot accounts -# OpenStack Release Bot: -## curl https://review.openstack.org/accounts/22816 -# OpenStack Proposal Bot -## curl https://review.openstack.org/accounts/11131 -venv/bin/python "./${OWNERS}" -a "${AFTER}" -b "${BEFORE}" \ - -i 11131 -i 22816 \ - -o "${TAG}" -r "${TAG}" -if [ "$WITH_STABLE" -eq 1 ] ; then - TMP_DIR=$(mktemp -d election.XXXXXXX) - echo "Starting (Stable) roll generation @" $(TZ=UTC date) - venv/bin/python "./${OWNERS}" -a "${AFTER}" -b "${BEFORE}" \ - -i 11131 -i 22816 \ - -o "${TMP_DIR}" -r "${TAG}" \ - -n -s 'branch:^stable/.*' - find - cp "${TMP_DIR}/_electorate.txt" "./${TAG}/stable_branch_maintenance.txt" - cp "${TMP_DIR}/_all_owners.yaml" "./${TAG}/stable_branch_maintenance.yaml" - echo "Finished (Stable) roll generation @" $(TZ=UTC date) - rm -rf "${TMP_DIR}" -else - echo Dropping stable_branch_maintenance data - rm "./${TAG}/stable_branch_maintenance.txt" \ - "./${TAG}/stable_branch_maintenance.yaml" -fi -echo "Finished roll generation @" $(TZ=UTC date) - -echo "Electoral rolls SHA256sum" -case "$(uname -s)" in - Darwin) - shasum -a 256 ./${TAG}/*.txt | tee "${TAG}.shasums" - ;; - *) - sha256sum ./${TAG}/*.txt | tee "${TAG}.shasums" - ;; -esac