Rewrite generate-rolls in python
Change-Id: I20de88a41eaf1d1bfba1e82f487ec508900f9494 Depends-On: Ie3b602237625e11651825abc8feb38caa73e9632
This commit is contained in:
Tristan Cacqueray
parent
20ca00cac5
commit
fe2ac4be95
113
openstack_election/cmds/generate_rolls.py
Executable file
113
openstack_election/cmds/generate_rolls.py
Executable file
@ -0,0 +1,113 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from __future__ import absolute_import
|
||||||
|
from __future__ import print_function
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import glob
|
||||||
|
import hashlib
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
|
||||||
|
from six.moves.urllib.request import urlopen
|
||||||
|
|
||||||
|
from openstack_election import utils
|
||||||
|
|
||||||
|
CGIT_URL = "http://git.openstack.org/cgit/openstack-infra/system-config/" \
|
||||||
|
"plain/tools/owners.py"
|
||||||
|
|
||||||
|
# Exclude the system / bot accounts
|
||||||
|
# OpenStack Release Bot:
|
||||||
|
# curl https://review.openstack.org/accounts/22816
|
||||||
|
# OpenStack Proposal Bot
|
||||||
|
# curl https://review.openstack.org/accounts/11131
|
||||||
|
MINUS_BOT_ACCOUNTS = ["-i", "11131", "-i", "22816"]
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
start = utils.conf['timeframe']['start']
|
||||||
|
end = utils.conf['timeframe']['end']
|
||||||
|
description = ('Generate electorate rolls')
|
||||||
|
parser = argparse.ArgumentParser(description)
|
||||||
|
parser.add_argument('--tag', dest='tag', default=utils.conf['tag'],
|
||||||
|
help=('The governance tag to validate against. '
|
||||||
|
'Default: %(default)s'))
|
||||||
|
parser.add_argument('--rolls_dir', default="./rolls/%s" % (
|
||||||
|
utils.conf['tag']), help=('The output directory. '
|
||||||
|
'Default: %(default)s'))
|
||||||
|
parser.add_argument('--before', default=end.strftime("%Y-%m-%d %H-%M-%S"),
|
||||||
|
help=('The start date. Default: %(default)s'))
|
||||||
|
parser.add_argument('--after', default=start.strftime("%Y-%m-%d"),
|
||||||
|
help=('The end date. Default: %(default)s'))
|
||||||
|
parser.add_argument('--without-stable', dest='with_stable', default=True,
|
||||||
|
action='store_false', help=('Do not include stables'))
|
||||||
|
parser.add_argument('--cached-owners-script', default=False,
|
||||||
|
action='store_true', help=('Do not fetch the script'))
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
print("Run information:")
|
||||||
|
print(" - TAG : %s" % args.tag)
|
||||||
|
print(" - ROLLS : %s" % args.rolls_dir)
|
||||||
|
print(" - STABLE : %s" % args.with_stable)
|
||||||
|
print(" - BEFORE : %s" % args.before)
|
||||||
|
print(" - AFTER : %s" % args.after)
|
||||||
|
|
||||||
|
if os.path.isdir(args.rolls_dir):
|
||||||
|
shutil.rmtree(args.rolls_dir)
|
||||||
|
os.makedirs(args.rolls_dir, 0o700)
|
||||||
|
|
||||||
|
os.chdir(os.path.dirname(args.rolls_dir))
|
||||||
|
if not args.cached_owners_script:
|
||||||
|
print("Grabbing script from: %s" % CGIT_URL)
|
||||||
|
with open("owners.py", "wb") as owner_file:
|
||||||
|
owner_file.write(urlopen(CGIT_URL).read())
|
||||||
|
|
||||||
|
sys.path.append(os.getcwd())
|
||||||
|
import owners
|
||||||
|
print("Starting roll generation @%s" % time.ctime())
|
||||||
|
owners.main(["owners.py", "-a", args.after, "-b", args.before,
|
||||||
|
"-o", args.tag, "-r", args.tag] + MINUS_BOT_ACCOUNTS)
|
||||||
|
print("Finished roll generation @%s" % time.ctime())
|
||||||
|
|
||||||
|
if args.with_stable:
|
||||||
|
tmp_dir = tempfile.mkdtemp(prefix='election.')
|
||||||
|
print("Starting (Stable) roll generation @%s" % time.ctime())
|
||||||
|
owners.main(["owners.py", "-a", args.after, "-b", args.before,
|
||||||
|
"-o", tmp_dir, "-r", args.tag, "-n",
|
||||||
|
"-s", "branch:^stable/.*"] + MINUS_BOT_ACCOUNTS)
|
||||||
|
print("Finished (Stable) roll generation @%s" % time.ctime())
|
||||||
|
shutil.copy("%s/_electorate.txt" % tmp_dir,
|
||||||
|
"./%s/stable_branch_maintenance.txt" % args.tag)
|
||||||
|
shutil.copy("%s/_all_owners.yaml" % tmp_dir,
|
||||||
|
"./%s/stable_branch_maintenance.yaml" % args.tag)
|
||||||
|
shutil.rmtree(tmp_dir)
|
||||||
|
|
||||||
|
print("Electoral rolls SHA256sum")
|
||||||
|
with open("%s.shasums" % args.tag, "wb") as ofile:
|
||||||
|
for fname in sorted(glob.glob("./%s/*.txt" % args.tag)):
|
||||||
|
# Sort the list
|
||||||
|
with open(fname) as list_file:
|
||||||
|
roll = sorted(list_file.readlines())
|
||||||
|
content = "".join(roll).encode('utf-8')
|
||||||
|
with open(fname, "wb") as list_file:
|
||||||
|
list_file.write(content)
|
||||||
|
# Generate the hash
|
||||||
|
h = hashlib.sha256(content).hexdigest()
|
||||||
|
s = "%s %s" % (h, fname)
|
||||||
|
print(s)
|
||||||
|
ofile.write(("%s\n" % s).encode('utf-8'))
|
||||||
@ -28,6 +28,7 @@ console_scripts =
|
|||||||
close-election = openstack_election.cmds.close_election:main
|
close-election = openstack_election.cmds.close_election:main
|
||||||
search-rolls = openstack_election.cmds.search_rolls:main
|
search-rolls = openstack_election.cmds.search_rolls:main
|
||||||
create-directories = openstack_election.cmds.create_directories:main
|
create-directories = openstack_election.cmds.create_directories:main
|
||||||
|
generate-rolls = openstack_election.cmds.generate_rolls:main
|
||||||
|
|
||||||
[build_sphinx]
|
[build_sphinx]
|
||||||
all_files = 1
|
all_files = 1
|
||||||
|
|||||||
@ -1,123 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
|
|
||||||
CGIT_URL=http://git.openstack.org/cgit/openstack-infra/system-config/plain/tools/owners.py
|
|
||||||
OWNERS=$(basename "$CGIT_URL")
|
|
||||||
WITH_STABLE=1
|
|
||||||
|
|
||||||
while [ $# -gt 0 ] ; do
|
|
||||||
case "$1" in
|
|
||||||
--tag)
|
|
||||||
TAG=$2
|
|
||||||
shift 1
|
|
||||||
;;
|
|
||||||
--before)
|
|
||||||
BEFORE=$2
|
|
||||||
shift 1
|
|
||||||
;;
|
|
||||||
--after)
|
|
||||||
AFTER=$2
|
|
||||||
shift 1
|
|
||||||
;;
|
|
||||||
--without-stable)
|
|
||||||
WITH_STABLE=0
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
shift 1
|
|
||||||
done
|
|
||||||
|
|
||||||
if [ -z "$TAG" ] ; then
|
|
||||||
echo $(basename "$0"): You must supply a --tag >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -z "$AFTER" ] ; then
|
|
||||||
echo $(basename "$0"): You must supply a --after >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -z "$BEFORE" ] ; then
|
|
||||||
echo $(basename "$0"): You must supply a --before >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
|
|
||||||
ROLLS_DIR="./rolls/${TAG}"
|
|
||||||
|
|
||||||
echo "Run information:"
|
|
||||||
echo " - TAG :" $TAG
|
|
||||||
echo " - ROLLS :" $ROLLS_DIR
|
|
||||||
echo " - STABLE :" $WITH_STABLE
|
|
||||||
echo " - AFTER :" $AFTER
|
|
||||||
echo " - BEFORE :" $BEFORE
|
|
||||||
|
|
||||||
rm -rf "${ROLLS_DIR}"
|
|
||||||
mkdir -p "${ROLLS_DIR}"
|
|
||||||
|
|
||||||
cd $(dirname "${ROLLS_DIR}")
|
|
||||||
|
|
||||||
# Building the venv isn't really needed everytime
|
|
||||||
if [ ! -d venv ] ; then
|
|
||||||
echo "Building tools venv"
|
|
||||||
(
|
|
||||||
# owners.py isn't python3 safe as string.maketrans is now
|
|
||||||
# str.maketrans
|
|
||||||
virtualenv -p python2 venv
|
|
||||||
./venv/bin/pip install -U pip setuptools wheel
|
|
||||||
./venv/bin/pip install pyyaml requests ndg-httpsclient
|
|
||||||
) >/dev/null 2>&1
|
|
||||||
if [ $? -ne 0 ] ; then
|
|
||||||
echo Creating Virtualenv failed >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
echo "Using existing venv"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Always get the newest script in case a bug has been fixed.
|
|
||||||
rm -f "./${OWNERS}" >/dev/null 2>&1
|
|
||||||
echo "Grabbing script from: ${CGIT_URL}"
|
|
||||||
wget "${CGIT_URL}" -O "./${OWNERS}" -o /dev/null
|
|
||||||
if [ ! -e "./${OWNERS}" ] ; then
|
|
||||||
echo Failed to download ${CGIT_URL} >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "Starting roll generation @" $(TZ=UTC date)
|
|
||||||
# Exclude the system / bot accounts
|
|
||||||
# OpenStack Release Bot:
|
|
||||||
## curl https://review.openstack.org/accounts/22816
|
|
||||||
# OpenStack Proposal Bot
|
|
||||||
## curl https://review.openstack.org/accounts/11131
|
|
||||||
venv/bin/python "./${OWNERS}" -a "${AFTER}" -b "${BEFORE}" \
|
|
||||||
-i 11131 -i 22816 \
|
|
||||||
-o "${TAG}" -r "${TAG}"
|
|
||||||
if [ "$WITH_STABLE" -eq 1 ] ; then
|
|
||||||
TMP_DIR=$(mktemp -d election.XXXXXXX)
|
|
||||||
echo "Starting (Stable) roll generation @" $(TZ=UTC date)
|
|
||||||
venv/bin/python "./${OWNERS}" -a "${AFTER}" -b "${BEFORE}" \
|
|
||||||
-i 11131 -i 22816 \
|
|
||||||
-o "${TMP_DIR}" -r "${TAG}" \
|
|
||||||
-n -s 'branch:^stable/.*'
|
|
||||||
find
|
|
||||||
cp "${TMP_DIR}/_electorate.txt" "./${TAG}/stable_branch_maintenance.txt"
|
|
||||||
cp "${TMP_DIR}/_all_owners.yaml" "./${TAG}/stable_branch_maintenance.yaml"
|
|
||||||
echo "Finished (Stable) roll generation @" $(TZ=UTC date)
|
|
||||||
rm -rf "${TMP_DIR}"
|
|
||||||
else
|
|
||||||
echo Dropping stable_branch_maintenance data
|
|
||||||
rm "./${TAG}/stable_branch_maintenance.txt" \
|
|
||||||
"./${TAG}/stable_branch_maintenance.yaml"
|
|
||||||
fi
|
|
||||||
echo "Finished roll generation @" $(TZ=UTC date)
|
|
||||||
|
|
||||||
echo "Electoral rolls SHA256sum"
|
|
||||||
case "$(uname -s)" in
|
|
||||||
Darwin)
|
|
||||||
shasum -a 256 ./${TAG}/*.txt | tee "${TAG}.shasums"
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
sha256sum ./${TAG}/*.txt | tee "${TAG}.shasums"
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
Loading…
Reference in New Issue
Block a user