Add opendev migration repo rename scripts
Git repo moves based on cgit aliases from project-config, the OpenStack TC guidance recorded in http://lists.openstack.org/pipermail/openstack-discuss/2019-April/004920.html and the ethercalc used to collect input from other users of the system. Also the results of an extensive bikeshedding session at http://eavesdrop.openstack.org/irclogs/%23openstack-infra/%23openstack-infra.2019-04-11.log.html#t2019-04-11T14:54:09 which concluded that anything left homeless goes in a namespace called "x" since that's short, a basic alphabetic character and provides no particular connotation. The opendev-migrate script, when run, provides a shareable rendering on stdout and also writes a repos.yaml file for input into the rename_repos playbook. The opendev-patching script, when run, uses the repos.yaml file and iterates over a tree of Git repositories updating their Zuul configuration, playbooks and roles as well as .gitreview files both for the project renames and the opendev hostname changes. It also creates a rename commit in project-config so that manage-projects will be in sync with the results of the rename_repos playbook. Change-Id: Ifa9fa6896110e8a33f32dcda6325bd58846935e2 Task: #30570 Co-Authored-By: James E. Blair <jeblair@redhat.com>
This commit is contained in:
Jeremy Stanley
parent
671250095d
commit
0c0b8e3087
128
tools/opendev-migrate
Normal file
128
tools/opendev-migrate
Normal file
@ -0,0 +1,128 @@
|
||||
#!/usr/bin/python3
|
||||
|
||||
# Copyright (c) 2019 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import requests
|
||||
import yaml
|
||||
|
||||
# this will hold our mapping of old names(paces) to new
|
||||
moves = {}
|
||||
|
||||
# here's a list of all (non-meta)projects in gerrit
|
||||
repos = [r for r in json.loads(requests.get(
|
||||
'http://review.openstack.org/projects/').text[5:]).keys() if '/' in r]
|
||||
|
||||
# a map of the first pair of columns from the namespace request ethercalc
|
||||
overrides = dict([r[:2] for r in csv.reader(io.StringIO(
|
||||
requests.get('https://ethercalc.openstack.org/opendev-transition.csv').text
|
||||
)) if '/' in r[1]])
|
||||
|
||||
# all projects which are officially governed by openstack or osf
|
||||
openstack = []
|
||||
o_gov = 'https://opendev.org/openstack/governance/raw/branch/master/reference/'
|
||||
data = yaml.safe_load(requests.get(o_gov + 'projects.yaml').text)
|
||||
for team in data.values():
|
||||
for deli in team['deliverables'].values():
|
||||
for repo in deli['repos']:
|
||||
openstack.append(repo)
|
||||
for f in ('foundation-board-repos.yaml', 'sigs-repos.yaml',
|
||||
'technical-committee-repos.yaml', 'user-committee-repos.yaml'):
|
||||
data = yaml.safe_load(requests.get(o_gov + f).text)
|
||||
for team in data.values():
|
||||
for repo in team:
|
||||
openstack.append(repo['repo'])
|
||||
|
||||
# projects which were at one time officially governed by openstack
|
||||
openstack_legacy = []
|
||||
data = yaml.safe_load(requests.get(o_gov + 'legacy.yaml').text)
|
||||
for team in data.values():
|
||||
for deli in team['deliverables'].values():
|
||||
for repo in deli['repos']:
|
||||
openstack_legacy.append(repo)
|
||||
|
||||
# use the jeepyb config to identify whitelabeled oip git projects
|
||||
airship = []
|
||||
starlingx = []
|
||||
zuul = []
|
||||
data = yaml.safe_load(requests.get(
|
||||
'https://opendev.org/openstack-infra/project-config/raw/branch/master/'
|
||||
'gerrit/projects.yaml').text)
|
||||
for project in data:
|
||||
if 'cgit-alias' in project:
|
||||
if project['cgit-alias']['site'] == 'git.airshipit.org':
|
||||
airship.append(project['project'])
|
||||
elif project['cgit-alias']['site'] == 'git.starlingx.io':
|
||||
starlingx.append(project['project'])
|
||||
elif project['cgit-alias']['site'] == 'git.zuul-ci.org':
|
||||
zuul.append(project['project'])
|
||||
|
||||
for repo in repos:
|
||||
# apply the requested namespace overrides first
|
||||
if repo in overrides:
|
||||
moves[repo] = overrides[repo]
|
||||
|
||||
# airship repos identified drop the airship- prefix and move to airship
|
||||
elif repo in airship:
|
||||
moves[repo] = 'airship/' + repo.split('/')[1].replace('airship-', '')
|
||||
|
||||
# starlingx repos drop the stx- prefix and move to starlingx
|
||||
elif repo in starlingx:
|
||||
moves[repo] = 'starlingx/' + repo.split('/')[1].replace('stx-', '')
|
||||
|
||||
# all current openstack repos move to openstack
|
||||
elif repo in openstack:
|
||||
moves[repo] = 'openstack/' + repo.split('/')[1]
|
||||
|
||||
# zuul repos move to zuul
|
||||
elif repo in zuul:
|
||||
moves[repo] = 'zuul/' + repo.split('/')[1]
|
||||
|
||||
# former openstack repositories which aren't accounted for go in openstack
|
||||
elif repo in openstack_legacy:
|
||||
moves[repo] = 'openstack/' + repo.split('/')[1]
|
||||
|
||||
# unofficial repositories move from openstack to x
|
||||
elif repo.startswith('openstack/'):
|
||||
moves[repo] = 'x/' + repo.split('/')[1]
|
||||
|
||||
# everything else is unchanged
|
||||
else:
|
||||
moves[repo] = repo
|
||||
|
||||
# we'll use this data structure for the rename_repos playbook input
|
||||
output = {'repos': []}
|
||||
|
||||
for mapping in moves.items():
|
||||
if mapping[0] != mapping[1]:
|
||||
# convenient stdout feedback is for sharing with people
|
||||
print('%s -> %s' % mapping)
|
||||
|
||||
# update the rename_repos data structure
|
||||
output['repos'].append({'old': mapping[0], 'new': mapping[1]})
|
||||
|
||||
# https://docs.openstack.org/infra/system-config/gerrit.html#renaming-a-project
|
||||
with open('repos.yaml', 'w') as outfile:
|
||||
yaml.dump(output, outfile)
|
||||
|
||||
# We should add this to the rename playbook, but time is short
|
||||
with open('zuul-rename.sh', 'w') as outfile:
|
||||
keyroot = '/var/lib/zuul/keys'
|
||||
for d in output['repos']:
|
||||
outfile.write('mv %s/ssh/project/gerrit/%s %s/ssh/project/gerrit/%s\n' %
|
||||
(keyroot, d['old'], keyroot, d['new']))
|
||||
outfile.write('mv %s/secrets/project/gerrit/%s %s/secrets/project/gerrit/%s\n' %
|
||||
(keyroot, d['old'], keyroot, d['new']))
|
||||
219
tools/opendev-patching
Normal file
219
tools/opendev-patching
Normal file
@ -0,0 +1,219 @@
|
||||
#!/usr/bin/python3
|
||||
|
||||
# Copyright (c) 2019 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
def run(commandlist):
|
||||
"""Wrapper to run a shell command and return a list of stdout lines."""
|
||||
(o, x) = subprocess.Popen(
|
||||
commandlist, env=gitenv, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL).communicate()
|
||||
return o.decode('utf-8').strip().split('\n')
|
||||
|
||||
|
||||
class EncryptedPKCS1_OAEP(yaml.YAMLObject):
|
||||
"""Causes pyyaml to skip custom YAML tags Zuul groks."""
|
||||
yaml_tag = u'!encrypted/pkcs1-oaep'
|
||||
yaml_loader = yaml.SafeLoader
|
||||
|
||||
def __init__(self, x):
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def from_yaml(cls, loader, node):
|
||||
return cls(node.value)
|
||||
|
||||
|
||||
# the gerrit git directory
|
||||
top = sys.argv[1]
|
||||
|
||||
# the repo renames file and a corresponding regex for finding them
|
||||
renames = {}
|
||||
for repo in yaml.safe_load(open(sys.argv[2]))['repos']:
|
||||
renames[repo['old']] = repo['new']
|
||||
renames_regex = re.compile(
|
||||
'([^a-z0-9_-]|^)(%s)([^a-z0-9_-]|$)' % '|'.join(renames.keys()))
|
||||
|
||||
# our custom git author/committer used by the run function
|
||||
gitenv = dict(os.environ)
|
||||
gitenv.update({
|
||||
'GIT_AUTHOR_NAME': 'OpenDev Sysadmins',
|
||||
'GIT_AUTHOR_EMAIL': 'openstack-infra@lists.openstack.org',
|
||||
'GIT_COMMITTER_NAME': 'OpenDev Sysadmins',
|
||||
'GIT_COMMITTER_EMAIL': 'openstack-infra@lists.openstack.org',
|
||||
})
|
||||
|
||||
# commit message string for generated commits
|
||||
commit_message = """\
|
||||
OpenDev Migration Patch
|
||||
|
||||
This commit was bulk generated and pushed by the OpenDev sysadmins
|
||||
as a part of the Git hosting and code review systems migration
|
||||
detailed in these mailing list posts:
|
||||
|
||||
http://lists.openstack.org/pipermail/openstack-discuss/2019-March/003603.html
|
||||
http://lists.openstack.org/pipermail/openstack-discuss/2019-April/004920.html
|
||||
|
||||
Attempts have been made to correct repository namespaces and
|
||||
hostnames based on simple pattern matching, but it's possible some
|
||||
were updated incorrectly or missed entirely. Please reach out to us
|
||||
via the contact information listed at https://opendev.org/ with any
|
||||
questions you may have.
|
||||
"""
|
||||
|
||||
# find all second-level directories on which we will operate
|
||||
repos = run(['find', top, '-maxdepth', '2', '-mindepth', '2', '-name', '*.git', '-type', 'd'])
|
||||
|
||||
# iterate over each repo
|
||||
for bare in repos:
|
||||
# clone the repo into a temporary working tree
|
||||
with tempfile.TemporaryDirectory() as repodir:
|
||||
run(['git', 'clone', bare, repodir])
|
||||
origdir = os.getcwd()
|
||||
os.chdir(repodir)
|
||||
|
||||
# build a list of branches for this repo
|
||||
branches = []
|
||||
branchdump = run(['git', 'branch', '-a'])
|
||||
|
||||
# iterate over each branch
|
||||
for line in branchdump:
|
||||
branch = re.match('^remotes/origin/([^ ]+)$', line.strip())
|
||||
if branch:
|
||||
branches.append(branch.group(1))
|
||||
for branch in branches:
|
||||
run(['git', 'checkout', '-B', branch, 'origin/' + branch])
|
||||
|
||||
# build up a list of files to edit
|
||||
editfiles = set()
|
||||
|
||||
# find zuul configs and add ansible playbooks they reference
|
||||
zuulfiles = run([
|
||||
'find', '.zuul.d/', 'zuul.d/', '.zuul.yaml', 'zuul.yaml',
|
||||
'-name', '*.yaml', '-type', 'f'])
|
||||
for zuulfile in zuulfiles:
|
||||
if zuulfile:
|
||||
conf = yaml.safe_load(open(zuulfile))
|
||||
if not conf:
|
||||
# some repos have empty zuul configs
|
||||
continue
|
||||
for node in conf:
|
||||
if 'job' in node:
|
||||
for subnode in ('post-run', 'pre-run', 'run'):
|
||||
if subnode in node['job']:
|
||||
if type(node['job'][subnode]) is list:
|
||||
editfiles.update(node['job'][subnode])
|
||||
else:
|
||||
editfiles.add(node['job'][subnode])
|
||||
|
||||
# if there are roles dirs relative to the playbooks, add them too
|
||||
for playbook in list(editfiles):
|
||||
rolesdir = os.path.join(os.path.dirname(playbook), 'roles')
|
||||
if os.path.isdir(rolesdir):
|
||||
editfiles.update(run([
|
||||
'find', rolesdir, '-type', 'f', '(', '-name', '*.j2',
|
||||
'-o', '-name', '*.yaml', '-o', '-name', '*.yml', ')']))
|
||||
|
||||
# zuul looks at the top level roles dir too
|
||||
editfiles.update(run([
|
||||
'find', 'roles', '-type', 'f', '(', '-name', '*.j2', '-o',
|
||||
'-name', '*.yaml', '-o', '-name', '*.yml', ')']))
|
||||
|
||||
# and add the zuul configs themselves
|
||||
editfiles.update(zuulfiles)
|
||||
|
||||
# and add .gitreview of course
|
||||
editfiles.add('.gitreview')
|
||||
|
||||
# and zuul/main.yaml so we catch the tenant config
|
||||
editfiles.add('zuul/main.yaml')
|
||||
|
||||
# and gerrit/projects.yaml for manage-projects
|
||||
editfiles.add('gerrit/projects.yaml')
|
||||
|
||||
# and gerritbot/channels.yaml for gerritbot
|
||||
editfiles.add('gerritbot/channels.yaml')
|
||||
|
||||
# drop any empty filename we ended up with
|
||||
editfiles.discard('')
|
||||
|
||||
# read through each file and replace specific patterns
|
||||
for fname in editfiles:
|
||||
if not os.path.exists(fname):
|
||||
continue
|
||||
with open(fname) as rfd, tempfile.NamedTemporaryFile() as wfd:
|
||||
# track modifications for efficiency
|
||||
modified = False
|
||||
for line in rfd:
|
||||
# apply renames from the mapping
|
||||
found = renames_regex.search(line)
|
||||
while found:
|
||||
line = line.replace(
|
||||
found.group(2), renames[found.group(2)])
|
||||
modified = True
|
||||
found = renames_regex.search(line)
|
||||
|
||||
# same for git.openstack.org -> opendev.org
|
||||
found = re.search("git\.openstack\.org", line)
|
||||
while found:
|
||||
line = line.replace(
|
||||
"git.openstack.org", "opendev.org")
|
||||
modified = True
|
||||
found = renames_regex.search(line)
|
||||
|
||||
# and review.openstack.org -> review.opendev.org
|
||||
found = re.search("review\.openstack\.org", line)
|
||||
while found:
|
||||
line = line.replace(
|
||||
"review.openstack.org", "review.opendev.org")
|
||||
modified = True
|
||||
found = renames_regex.search(line)
|
||||
|
||||
wfd.write(line.encode('utf-8'))
|
||||
|
||||
# copy any modified file back into the worktree
|
||||
if modified:
|
||||
wfd.flush()
|
||||
shutil.copyfile(wfd.name, fname)
|
||||
modified = False
|
||||
|
||||
# special logic to rename Gerrit ACL files
|
||||
if bare.endswith('/project-config.git'):
|
||||
for acl in run(['git', 'ls-files', 'gerrit/acls/']):
|
||||
found = renames_regex.search(acl)
|
||||
if found:
|
||||
newpath = acl.replace(
|
||||
found.group(2), renames[found.group(2)])
|
||||
os.makedirs(os.path.dirname(newpath), exist_ok=True)
|
||||
run(['git', 'mv', acl, newpath])
|
||||
|
||||
# commit and push our changes, if there are any
|
||||
if run(['git', 'diff']):
|
||||
with tempfile.NamedTemporaryFile() as message:
|
||||
message.write(commit_message.encode('utf-8'))
|
||||
message.flush()
|
||||
run(['git', 'commit', '-a', '-F', message.name])
|
||||
run(['git', 'push', 'origin', 'HEAD'])
|
||||
|
||||
# switch back before the context manager deletes our cwd
|
||||
os.chdir(origdir)
|
||||
Loading…
x
Reference in New Issue
Block a user