From af48c6d986df63afa73c1b99614b4607843040b4 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 1 May 2012 17:11:48 -0400 Subject: [PATCH] Move gerrit scripts to puppet. Take the things from openstack-ci/gerrit and move them directly in to the puppet module. Install them using the model we're using for the jenkins slave scripts. Change-Id: I420b2b895bd57d40232b2cdda437617373a82890 --- modules/gerrit/files/change-merged | 2 +- modules/gerrit/files/patchset-created | 4 +- .../files/scripts/close_pull_requests.py | 78 ++++ .../files/scripts/expire_old_reviews.py | 74 +++ .../gerrit/files/scripts/get_group_uuid.py | 29 ++ .../gerrit/files/scripts/update_blueprint.py | 139 ++++++ modules/gerrit/files/scripts/update_bug.py | 200 ++++++++ .../gerrit/files/scripts/update_cla_group.py | 77 ++++ .../files/scripts/update_gerrit_users.py | 391 ++++++++++++++++ modules/gerrit/files/scripts/update_users.py | 430 ++++++++++++++++++ modules/gerrit/manifests/init.pp | 33 +- 11 files changed, 1446 insertions(+), 11 deletions(-) create mode 100755 modules/gerrit/files/scripts/close_pull_requests.py create mode 100644 modules/gerrit/files/scripts/expire_old_reviews.py create mode 100644 modules/gerrit/files/scripts/get_group_uuid.py create mode 100755 modules/gerrit/files/scripts/update_blueprint.py create mode 100755 modules/gerrit/files/scripts/update_bug.py create mode 100755 modules/gerrit/files/scripts/update_cla_group.py create mode 100755 modules/gerrit/files/scripts/update_gerrit_users.py create mode 100644 modules/gerrit/files/scripts/update_users.py diff --git a/modules/gerrit/files/change-merged b/modules/gerrit/files/change-merged index 61e8610ae2..7516919104 100755 --- a/modules/gerrit/files/change-merged +++ b/modules/gerrit/files/change-merged @@ -1,3 +1,3 @@ #!/bin/sh -python ~/openstack-ci/gerrit/update_bug.py change-merged "$@" +python /usr/local/gerrit/scripts/update_bug.py change-merged "$@" diff --git a/modules/gerrit/files/patchset-created b/modules/gerrit/files/patchset-created index 68d66fc444..23d4ce9ba4 100755 --- a/modules/gerrit/files/patchset-created +++ b/modules/gerrit/files/patchset-created @@ -1,4 +1,4 @@ #!/bin/sh -python ~/openstack-ci/gerrit/update_blueprint.py patchset-created "$@" -python ~/openstack-ci/gerrit/update_bug.py patchset-created "$@" +python /usr/local/gerrit/scripts/update_blueprint.py patchset-created "$@" +python /usr/local/gerrit/scripts/update_bug.py patchset-created "$@" diff --git a/modules/gerrit/files/scripts/close_pull_requests.py b/modules/gerrit/files/scripts/close_pull_requests.py new file mode 100755 index 0000000000..0355e90d48 --- /dev/null +++ b/modules/gerrit/files/scripts/close_pull_requests.py @@ -0,0 +1,78 @@ +#! /usr/bin/env python +# Copyright (C) 2011 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Close Github pull requests with instructions to use Gerrit for +# code review. The list of projects is found in github.config +# and should look like: + +# [project "GITHUB_PROJECT"] +# close_pull = true + +# Github authentication information is read from github.secure.config, +# which should look like: + +# [github] +# username = GITHUB_USERNAME +# api_token = GITHUB_API_TOKEN + +import github2.client +import os +import StringIO +import ConfigParser +import logging +import re + +logging.basicConfig(level=logging.ERROR) + +GITHUB_CONFIG = os.environ.get('GITHUB_CONFIG', + '/home/gerrit2/github.config') +GITHUB_SECURE_CONFIG = os.environ.get('GITHUB_SECURE_CONFIG', + '/home/gerrit2/github.secure.config') + +MESSAGE = """Thank you for contributing to OpenStack! + +%(project)s uses Gerrit for code review. + +Please visit http://wiki.openstack.org/GerritWorkflow and follow the instructions there to upload your change to Gerrit. +""" + +PROJECT_RE = re.compile(r'^project\s+"(.*)"$') + +secure_config = ConfigParser.ConfigParser() +secure_config.read(GITHUB_SECURE_CONFIG) +config = ConfigParser.ConfigParser() +config.read(GITHUB_CONFIG) + +github = github2.client.Github(requests_per_second=1.0, + username=secure_config.get("github", "username"), + api_token=secure_config.get("github", "api_token")) + +for section in config.sections(): + # Each section looks like [project "openstack/project"] + m = PROJECT_RE.match(section) + if not m: continue + project = m.group(1) + + # Make sure we're supposed to close pull requests for this project: + if not (config.has_option(section, "close_pull") and + config.get(section, "close_pull").lower() == 'true'): + continue + + # Close each pull request + pull_requests = github.pull_requests.list(project) + for req in pull_requests: + vars = dict(project=project) + github.issues.comment(project, req.number, MESSAGE%vars) + github.issues.close(project, req.number) diff --git a/modules/gerrit/files/scripts/expire_old_reviews.py b/modules/gerrit/files/scripts/expire_old_reviews.py new file mode 100644 index 0000000000..63b8352d9a --- /dev/null +++ b/modules/gerrit/files/scripts/expire_old_reviews.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python +# Copyright (c) 2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This script is designed to expire old code reviews that have not been touched +# using the following rules: +# 1. if open and no activity in 2 weeks, expire +# 2. if negative comment and no activity in 1 week, expire + +import os +import paramiko +import json +import logging + +GERRIT_USER = os.environ.get('GERRIT_USER', 'launchpadsync') +GERRIT_SSH_KEY = os.environ.get('GERRIT_SSH_KEY', + '/home/gerrit2/.ssh/launchpadsync_rsa') + +logging.basicConfig(format='%(asctime)-6s: %(name)s - %(levelname)s - %(message)s', filename='/var/log/gerrit/expire_reviews.log') +logger= logging.getLogger('expire_reviews') +logger.setLevel(logging.INFO) + +logger.info('Starting expire reviews') +logger.info('Connecting to Gerrit') + +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +ssh.connect('localhost', username=GERRIT_USER, key_filename=GERRIT_SSH_KEY, port=29418) + +def expire_patch_set(patch_id, patch_subject, has_negative): + if has_negative: + message= 'code review expired after 1 week of no activity after a negative review' + else: + message= 'code review expired after 2 weeks of no activity' + command='gerrit review --abandon --message="{0}" {1}'.format(message, patch_id) + logger.info('Expiring: %s - %s: %s', patch_id, patch_subject, message) + stdin, stdout, stderr = ssh.exec_command(command) + if stdout.channel.recv_exit_status() != 0: + logger.error(stderr.read()) + +# Query all open with no activity for 2 weeks +logger.info('Searching no activity for 2 weeks') +stdin, stdout, stderr = ssh.exec_command('gerrit query --current-patch-set --format JSON status:open age:2w') + +for line in stdout: + row= json.loads(line) + if not row.has_key('rowCount'): + expire_patch_set(row['currentPatchSet']['revision'], row['subject'], False) + +# Query all reviewed with no activity for 1 week +logger.info('Searching no activity on negative review for 1 week') +stdin, stdout, stderr = ssh.exec_command('gerrit query --current-patch-set --all-approvals --format JSON status:reviewed age:1w') + +for line in stdout: + row= json.loads(line) + if not row.has_key('rowCount'): + # Search for negative approvals + for approval in row['currentPatchSet']['approvals']: + if approval['value'] == '-1': + expire_patch_set(row['currentPatchSet']['revision'], row['subject'], True) + break + +logger.info('End expire review') diff --git a/modules/gerrit/files/scripts/get_group_uuid.py b/modules/gerrit/files/scripts/get_group_uuid.py new file mode 100644 index 0000000000..335e944089 --- /dev/null +++ b/modules/gerrit/files/scripts/get_group_uuid.py @@ -0,0 +1,29 @@ +import argparse +import paramiko +import json + +parser = argparse.ArgumentParser() +parser.add_argument("--host", dest="host", default="review.openstack.org", + help="gerrit host to connect to") +parser.add_argument("--port", dest="port", action='store', type=int, + default=29418, help="gerrit port to connect to") +parser.add_argument("groups", nargs=1) + +options = parser.parse_args() + + +client = paramiko.SSHClient() +client.load_system_host_keys() +client.set_missing_host_key_policy(paramiko.WarningPolicy()) +client.connect(options.host, port=options.port) + +group = options.groups[0] +query = "select group_uuid from account_groups where name = '%s'" % group +command = 'gerrit gsql --format JSON -c "%s"' % query +stdin, stdout, stderr = client.exec_command(command) + +for line in stdout: + row = json.loads(line) + if row['type'] == 'row': + print row['columns']['group_uuid'] + ret = stdout.channel.recv_exit_status() diff --git a/modules/gerrit/files/scripts/update_blueprint.py b/modules/gerrit/files/scripts/update_blueprint.py new file mode 100755 index 0000000000..9c18f1d4f9 --- /dev/null +++ b/modules/gerrit/files/scripts/update_blueprint.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python +# Copyright (c) 2011 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This is designed to be called by a gerrit hook. It searched new +# patchsets for strings like "blueprint FOO" or "bp FOO" and updates +# corresponding Launchpad blueprints with links back to the change. + +from launchpadlib.launchpad import Launchpad +from launchpadlib.uris import LPNET_SERVICE_ROOT +import os +import argparse +import re +import subprocess + +import StringIO +import ConfigParser +import MySQLdb + +BASE_DIR = '/home/gerrit2/review_site' +GERRIT_CACHE_DIR = os.path.expanduser(os.environ.get('GERRIT_CACHE_DIR', + '~/.launchpadlib/cache')) +GERRIT_CREDENTIALS = os.path.expanduser(os.environ.get('GERRIT_CREDENTIALS', + '~/.launchpadlib/creds')) +GERRIT_CONFIG = os.environ.get('GERRIT_CONFIG', + '/home/gerrit2/review_site/etc/gerrit.config') +GERRIT_SECURE_CONFIG = os.environ.get('GERRIT_SECURE_CONFIG', + '/home/gerrit2/review_site/etc/secure.config') +SPEC_RE = re.compile(r'(blueprint|bp)\s*[#:]?\s*(\S+)', re.I) +BODY_RE = re.compile(r'^\s+.*$') + +def get_broken_config(filename): + """ gerrit config ini files are broken and have leading tabs """ + text = "" + with open(filename,"r") as conf: + for line in conf.readlines(): + text = "%s%s" % (text, line.lstrip()) + + fp = StringIO.StringIO(text) + c=ConfigParser.ConfigParser() + c.readfp(fp) + return c + +GERRIT_CONFIG = get_broken_config(GERRIT_CONFIG) +SECURE_CONFIG = get_broken_config(GERRIT_SECURE_CONFIG) +DB_USER = GERRIT_CONFIG.get("database", "username") +DB_PASS = SECURE_CONFIG.get("database","password") +DB_DB = GERRIT_CONFIG.get("database","database") + +def update_spec(launchpad, project, name, subject, link, topic=None): + # For testing, if a project doesn't match openstack/foo, use + # the openstack-ci project instead. + group, project = project.split('/') + if group != 'openstack': + project = 'openstack-ci' + + spec = launchpad.projects[project].getSpecification(name=name) + if not spec: return + + if spec.whiteboard: + wb = spec.whiteboard.strip() + else: + wb = '' + changed = False + if topic: + topiclink = '%s/#q,topic:%s,n,z' % (link[:link.find('/',8)], + topic) + if topiclink not in wb: + wb += "\n\n\nGerrit topic: %(link)s" % dict(link=topiclink) + changed = True + + if link not in wb: + wb += "\n\n\nAddressed by: %(link)s\n %(subject)s\n" % dict(subject=subject, + link=link) + changed = True + + if changed: + spec.whiteboard = wb + spec.lp_save() + +def find_specs(launchpad, dbconn, args): + git_log = subprocess.Popen(['git', + '--git-dir=' + BASE_DIR + '/git/' + args.project + '.git', + 'log', '--no-merges', + args.commit + '^1..' + args.commit], + stdout=subprocess.PIPE).communicate()[0] + + cur = dbconn.cursor() + cur.execute("select subject, topic from changes where change_key=%s", args.change) + subject, topic = cur.fetchone() + specs = set([m.group(2) for m in SPEC_RE.finditer(git_log)]) + + if topic: + topicspec = topic.split('/')[-1] + specs |= set([topicspec]) + + for spec in specs: + update_spec(launchpad, args.project, spec, subject, + args.change_url, topic) + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('hook') + #common + parser.add_argument('--change', default=None) + parser.add_argument('--change-url', default=None) + parser.add_argument('--project', default=None) + parser.add_argument('--branch', default=None) + parser.add_argument('--commit', default=None) + #change-merged + parser.add_argument('--submitter', default=None) + # patchset-created + parser.add_argument('--uploader', default=None) + parser.add_argument('--patchset', default=None) + + args = parser.parse_args() + + launchpad = Launchpad.login_with('Gerrit User Sync', LPNET_SERVICE_ROOT, + GERRIT_CACHE_DIR, + credentials_file = GERRIT_CREDENTIALS, + version='devel') + + conn = MySQLdb.connect(user = DB_USER, passwd = DB_PASS, db = DB_DB) + + find_specs(launchpad, conn, args) + +if __name__ == '__main__': + main() diff --git a/modules/gerrit/files/scripts/update_bug.py b/modules/gerrit/files/scripts/update_bug.py new file mode 100755 index 0000000000..8afcb3bddb --- /dev/null +++ b/modules/gerrit/files/scripts/update_bug.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python +# Copyright (c) 2011 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This is designed to be called by a gerrit hook. It searched new +# patchsets for strings like "bug FOO" and updates corresponding Launchpad +# bugs status. + +from launchpadlib.launchpad import Launchpad +from launchpadlib.uris import LPNET_SERVICE_ROOT +import os +import argparse +import re +import subprocess + + +BASE_DIR = '/home/gerrit2/review_site' +GERRIT_CACHE_DIR = os.path.expanduser(os.environ.get('GERRIT_CACHE_DIR', + '~/.launchpadlib/cache')) +GERRIT_CREDENTIALS = os.path.expanduser(os.environ.get('GERRIT_CREDENTIALS', + '~/.launchpadlib/creds')) + + +def add_change_proposed_message(bugtask, change_url, project, branch): + subject = 'Fix proposed to %s (%s)' % (short_project(project), branch) + body = 'Fix proposed to branch: %s\nReview: %s' % (branch, change_url) + bugtask.bug.newMessage(subject=subject, content=body) + + +def add_change_merged_message(bugtask, change_url, project, commit, + submitter, branch, git_log): + subject = 'Fix merged to %s (%s)' % (short_project(project), branch) + git_url = 'http://github.com/%s/commit/%s' % (project, commit) + body = '''Reviewed: %s +Committed: %s +Submitter: %s +Branch: %s\n''' % (change_url, git_url, submitter, branch) + body = body + '\n' + git_log + bugtask.bug.newMessage(subject=subject, content=body) + + +def set_in_progress(bugtask, launchpad, uploader, change_url): + """Set bug In progress with assignee being the uploader""" + + # Retrieve uploader from Launchpad. Use email as search key if + # provided, and only set if there is a clear match. + try: + searchkey = uploader[uploader.rindex("(") + 1:-1] + except ValueError: + searchkey = uploader + persons = launchpad.people.findPerson(text=searchkey) + if len(persons) == 1: + bugtask.assignee = persons[0] + + bugtask.status = "In Progress" + bugtask.lp_save() + + +def set_fix_committed(bugtask): + """Set bug fix committed""" + + bugtask.status = "Fix Committed" + bugtask.lp_save() + + +def release_fixcommitted(bugtask): + """Set bug FixReleased if it was FixCommitted""" + + if bugtask.status == u'Fix Committed': + bugtask.status = "Fix Released" + bugtask.lp_save() + + +def tag_in_branchname(bugtask, branch): + """Tag bug with in-branch-name tag (if name is appropriate)""" + + lp_bug = bugtask.bug + branch_name = branch.replace('/', '-') + if branch_name.replace('-', '').isalnum(): + lp_bug.tags = lp_bug.tags + ["in-%s" % branch_name] + lp_bug.tags.append("in-%s" % branch_name) + lp_bug.lp_save() + + +def short_project(full_project_name): + """Return the project part of the git repository name""" + return full_project_name.split('/')[-1] + + +def git2lp(full_project_name): + """Convert Git repo name to Launchpad project""" + project_map = { + 'openstack/python-glanceclient': 'glance', + 'openstack/python-keystoneclient': 'keystone', + 'openstack/python-melangeclient': 'melange', + 'openstack/python-novaclient': 'nova', + 'openstack/python-quantumclient': 'quantum', + 'openstack/openstack-ci-puppet': 'openstack-ci', + 'openstack-ci/devstack-gate': 'openstack-ci', + } + return project_map.get(full_project_name, short_project(full_project_name)) + + +def process_bugtask(launchpad, bugtask, git_log, args): + """Apply changes to bugtask, based on hook / branch...""" + + if args.hook == "change-merged": + if args.branch == 'master': + set_fix_committed(bugtask) + elif args.branch == 'milestone-proposed': + release_fixcommitted(bugtask) + else: + tag_in_branchname(bugtask, args.branch) + add_change_merged_message(bugtask, args.change_url, args.project, + args.commit, args.submitter, args.branch, + git_log) + + if args.hook == "patchset-created": + if args.branch == 'master': + set_in_progress(bugtask, launchpad, args.uploader, args.change_url) + if args.patchset == '1': + add_change_proposed_message(bugtask, args.change_url, + args.project, args.branch) + + +def find_bugs(launchpad, git_log, args): + """Find bugs referenced in the git log and return related bugtasks""" + + bug_regexp = r'([Bb]ug|[Ll][Pp])[\s#:]*(\d+)' + tokens = re.split(bug_regexp, git_log) + + # Extract unique bug tasks + bugtasks = {} + for token in tokens: + if re.match('^\d+$', token) and (token not in bugtasks): + try: + lp_bug = launchpad.bugs[token] + for lp_task in lp_bug.bug_tasks: + if lp_task.bug_target_name == git2lp(args.project): + bugtasks[token] = lp_task + break + except KeyError: + # Unknown bug + pass + + return bugtasks.values() + + +def extract_git_log(args): + """Extract git log of all merged commits""" + cmd = ['git', + '--git-dir=' + BASE_DIR + '/git/' + args.project + '.git', + 'log', '--no-merges', args.commit + '^1..' + args.commit] + return subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0] + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('hook') + #common + parser.add_argument('--change', default=None) + parser.add_argument('--change-url', default=None) + parser.add_argument('--project', default=None) + parser.add_argument('--branch', default=None) + parser.add_argument('--commit', default=None) + #change-merged + parser.add_argument('--submitter', default=None) + #patchset-created + parser.add_argument('--uploader', default=None) + parser.add_argument('--patchset', default=None) + + args = parser.parse_args() + + # Connect to Launchpad + launchpad = Launchpad.login_with('Gerrit User Sync', LPNET_SERVICE_ROOT, + GERRIT_CACHE_DIR, + credentials_file=GERRIT_CREDENTIALS, + version='devel') + + # Get git log + git_log = extract_git_log(args) + + # Process bugtasks found in git log + for bugtask in find_bugs(launchpad, git_log, args): + process_bugtask(launchpad, bugtask, git_log, args) + + +if __name__ == '__main__': + main() diff --git a/modules/gerrit/files/scripts/update_cla_group.py b/modules/gerrit/files/scripts/update_cla_group.py new file mode 100755 index 0000000000..91365a2fa4 --- /dev/null +++ b/modules/gerrit/files/scripts/update_cla_group.py @@ -0,0 +1,77 @@ +#! /usr/bin/env python +# Copyright (C) 2011 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Add launchpad ids listed in the wiki CLA page to the CLA group in LP. + +import os +import sys +import uuid +import os +import urllib +import re + +import StringIO +import ConfigParser + +from launchpadlib.launchpad import Launchpad +from launchpadlib.uris import LPNET_SERVICE_ROOT + +DEBUG = False + +LP_CACHE_DIR = '~/.launchpadlib/cache' +LP_CREDENTIALS = '~/.launchpadlib/creds' +CONTRIBUTOR_RE = re.compile(r'.*?\|\|\s*(?P.*?)\s*\|\|\s*(?P.*?)\s*\|\|\s*(?P.*?)\s*\|\|.*?') +LINK_RE = re.compile(r'\[\[.*\|\s*(?P.*)\s*\]\]') + +for check_path in (os.path.dirname(LP_CACHE_DIR), + os.path.dirname(LP_CREDENTIALS)): + if not os.path.exists(check_path): + os.makedirs(check_path) + +wiki_members = [] +for line in urllib.urlopen('http://wiki.openstack.org/Contributors?action=raw'): + m = CONTRIBUTOR_RE.match(line) + if m and m.group('login') and m.group('trans'): + login = m.group('login') + if login=="<#c0c0c0>'''Launchpad ID'''": continue + l = LINK_RE.match(login) + if l: + login = l.group('name') + wiki_members.append(login) + +launchpad = Launchpad.login_with('CLA Team Sync', LPNET_SERVICE_ROOT, + LP_CACHE_DIR, + credentials_file = LP_CREDENTIALS) + +lp_members = [] + +team = launchpad.people['openstack-cla'] +for detail in team.members_details: + user = None + # detail.self_link == + # 'https://api.launchpad.net/1.0/~team/+member/${username}' + login = detail.self_link.split('/')[-1] + status = detail.status + lp_members.append(login) + +for wm in wiki_members: + if wm not in lp_members: + print "Need to add %s to LP" % (wm) + try: + person = launchpad.people[wm] + except: + print 'Unable to find %s on LP'%wm + continue + status = team.addMember(person=person, status="Approved") diff --git a/modules/gerrit/files/scripts/update_gerrit_users.py b/modules/gerrit/files/scripts/update_gerrit_users.py new file mode 100755 index 0000000000..11b1054a15 --- /dev/null +++ b/modules/gerrit/files/scripts/update_gerrit_users.py @@ -0,0 +1,391 @@ +#! /usr/bin/env python +# Copyright (C) 2011 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Synchronize Gerrit users from Launchpad. + +import os +import sys +import uuid +import os +import subprocess + +from datetime import datetime + +import StringIO +import ConfigParser + +import MySQLdb + +from launchpadlib.launchpad import Launchpad +from launchpadlib.uris import LPNET_SERVICE_ROOT + +from openid.consumer import consumer +from openid.cryptutil import randomString + +DEBUG = False + +GERRIT_USER = os.environ.get('GERRIT_USER', 'launchpadsync') +GERRIT_CONFIG = os.environ.get('GERRIT_CONFIG', + '/home/gerrit2/review_site/etc/gerrit.config') +GERRIT_SECURE_CONFIG = os.environ.get('GERRIT_SECURE_CONFIG', + '/home/gerrit2/review_site/etc/secure.config') +GERRIT_SSH_KEY = os.environ.get('GERRIT_SSH_KEY', + '/home/gerrit2/.ssh/launchpadsync_rsa') +GERRIT_CACHE_DIR = os.path.expanduser(os.environ.get('GERRIT_CACHE_DIR', + '~/.launchpadlib/cache')) +GERRIT_CREDENTIALS = os.path.expanduser(os.environ.get('GERRIT_CREDENTIALS', + '~/.launchpadlib/creds')) +GERRIT_BACKUP_PATH = os.environ.get('GERRIT_BACKUP_PATH', + '/home/gerrit2/dbupdates') + +for check_path in (os.path.dirname(GERRIT_CACHE_DIR), + os.path.dirname(GERRIT_CREDENTIALS), + GERRIT_BACKUP_PATH): + if not os.path.exists(check_path): + os.makedirs(check_path) + +def get_broken_config(filename): + """ gerrit config ini files are broken and have leading tabs """ + text = "" + with open(filename,"r") as conf: + for line in conf.readlines(): + text = "%s%s" % (text, line.lstrip()) + + fp = StringIO.StringIO(text) + c=ConfigParser.ConfigParser() + c.readfp(fp) + return c + +def get_type(in_type): + if in_type == "RSA": + return "ssh-rsa" + else: + return "ssh-dsa" + +gerrit_config = get_broken_config(GERRIT_CONFIG) +secure_config = get_broken_config(GERRIT_SECURE_CONFIG) + +DB_USER = gerrit_config.get("database", "username") +DB_PASS = secure_config.get("database","password") +DB_DB = gerrit_config.get("database","database") + +db_backup_file = "%s.%s.sql" % (DB_DB, datetime.isoformat(datetime.now())) +db_backup_path = os.path.join(GERRIT_BACKUP_PATH, db_backup_file) +retval = os.system("mysqldump --opt -u%s -p%s %s > %s" % + (DB_USER, DB_PASS, DB_DB, db_backup_path)) +if retval != 0: + print "Problem taking a db dump, aborting db update" + sys.exit(retval) + +conn = MySQLdb.connect(user = DB_USER, passwd = DB_PASS, db = DB_DB) +cur = conn.cursor() + + +launchpad = Launchpad.login_with('Gerrit User Sync', LPNET_SERVICE_ROOT, + GERRIT_CACHE_DIR, + credentials_file = GERRIT_CREDENTIALS) + +def get_sub_teams(team, have_teams): + for sub_team in launchpad.people[team].sub_teams: + if sub_team.name not in have_teams: + have_teams = get_sub_teams(sub_team.name, have_teams) + have_teams.append(team) + return have_teams + + +teams_todo = get_sub_teams('openstack', []) + +users={} +groups={} +groups_in_groups={} +group_implies_groups={} +group_ids={} +projects = subprocess.check_output(['/usr/bin/ssh', '-p', '29418', + '-i', GERRIT_SSH_KEY, + '-l', GERRIT_USER, 'localhost', + 'gerrit', 'ls-projects']).split('\n') + +for team_todo in teams_todo: + + team = launchpad.people[team_todo] + groups[team.name] = team.display_name + + # Attempt to get nested group memberships. ~nova-core, for instance, is a + # member of ~nova, so membership in ~nova-core should imply membership in + # ~nova + group_in_group = groups_in_groups.get(team.name, {}) + for subgroup in team.sub_teams: + group_in_group[subgroup.name] = 1 + # We should now have a dictionary of the form {'nova': {'nova-core': 1}} + groups_in_groups[team.name] = group_in_group + + for detail in team.members_details: + + user = None + + # detail.self_link == + # 'https://api.launchpad.net/1.0/~team/+member/${username}' + login = detail.self_link.split('/')[-1] + + if users.has_key(login): + user = users[login] + else: + + user = dict(add_groups=[]) + + status = detail.status + if (status == "Approved" or status == "Administrator"): + user['add_groups'].append(team.name) + users[login] = user + +# If we picked up subgroups that were not in our original list of groups +# make sure they get added +for (supergroup, subgroups) in groups_in_groups.items(): + for group in subgroups.keys(): + if group not in groups.keys(): + groups[group] = None + +# account_groups +# groups is a dict of team name to team display name +# here, for every group we have in that dict, we're building another dict of +# group_name to group_id - and if the database doesn't already have the +# group, we're adding it +for (group_name, group_display_name) in groups.items(): + if cur.execute("select group_id from account_groups where name = %s", + group_name): + group_ids[group_name] = cur.fetchall()[0][0] + else: + cur.execute("""insert into account_group_id (s) values (NULL)"""); + cur.execute("select max(s) from account_group_id") + group_id = cur.fetchall()[0][0] + + # Match the 40-char 'uuid' that java is producing + group_uuid = uuid.uuid4() + second_uuid = uuid.uuid4() + full_uuid = "%s%s" % (group_uuid.hex, second_uuid.hex[:8]) + + cur.execute("""insert into account_groups + (group_id, group_type, owner_group_id, + name, description, group_uuid) + values + (%s, 'INTERNAL', 1, %s, %s, %s)""", + (group_id, group_name, group_display_name, full_uuid)) + cur.execute("""insert into account_group_names (group_id, name) values + (%s, %s)""", + (group_id, group_name)) + + group_ids[group_name] = group_id + +# account_group_includes +# groups_in_groups should be a dict of dicts, where the key is the larger +# group and the inner dict is a list of groups that are members of the +# larger group. So {'nova': {'nova-core': 1}} +for (group_name, subgroups) in groups_in_groups.items(): + for subgroup_name in subgroups.keys(): + try: + cur.execute("""insert into account_group_includes + (group_id, include_id) + values (%s, %s)""", + (group_ids[group_name], group_ids[subgroup_name])) + except MySQLdb.IntegrityError: + pass + +# Make a list of implied group membership +# building a list which is the opposite of groups_in_group. Here +# group_implies_groups is a dict keyed by group_id containing a list of +# group_ids of implied membership. SO: if nova is 1 and nova-core is 2: +# {'2': [1]} +for group_id in group_ids.values(): + total_groups = [] + groups_todo = [group_id] + while len(groups_todo) > 0: + current_group = groups_todo.pop() + total_groups.append(current_group) + cur.execute("""select group_id from account_group_includes + where include_id = %s""", (current_group)) + for row in cur.fetchall(): + if row[0] != 1 and row[0] not in total_groups: + groups_todo.append(row[0]) + group_implies_groups[group_id] = total_groups + +if DEBUG: + def get_group_name(in_group_id): + for (group_name, group_id) in group_ids.items(): + if group_id == in_group_id: + return group_name + + print "groups in groups" + for (k,v) in groups_in_groups.items(): + print k, v + + print "group_imples_groups" + for (k, v) in group_implies_groups.items(): + print get_group_name(k) + new_groups=[] + for val in v: + new_groups.append(get_group_name(val)) + print "\t", new_groups + +for (username, user_details) in users.items(): + + # accounts + account_id = None + if cur.execute("""select account_id from account_external_ids where + external_id in (%s)""", ("username:%s" % username)): + account_id = cur.fetchall()[0][0] + # We have this bad boy - all we need to do is update his group membership + + else: + + # We need details + member = launchpad.people[username] + if not member.is_team: + + openid_consumer = consumer.Consumer(dict(id=randomString(16, '0123456789abcdef')), None) + openid_request = openid_consumer.begin("https://launchpad.net/~%s" % member.name) + user_details['openid_external_id'] = openid_request.endpoint.getLocalID() + + # Handle username change + if cur.execute("""select account_id from account_external_ids where + external_id in (%s)""", user_details['openid_external_id']): + account_id = cur.fetchall()[0][0] + cur.execute("""update account_external_ids + set external_id=%s + where external_id like 'username%%' + and account_id = %s""", + ('username:%s' % username, account_id)) + else: + user_details['ssh_keys'] = ["%s %s %s" % (get_type(key.keytype), key.keytext, key.comment) for key in member.sshkeys] + + + email = None + try: + email = member.preferred_email_address.email + except ValueError: + pass + user_details['email'] = email + + + cur.execute("""insert into account_id (s) values (NULL)"""); + cur.execute("select max(s) from account_id") + account_id = cur.fetchall()[0][0] + + cur.execute("""insert into accounts (account_id, full_name, preferred_email) values + (%s, %s, %s)""", (account_id, username, user_details['email'])) + + # account_ssh_keys + for key in user_details['ssh_keys']: + + cur.execute("""select ssh_public_key from account_ssh_keys where + account_id = %s""", account_id) + db_keys = [r[0].strip() for r in cur.fetchall()] + if key.strip() not in db_keys: + + cur.execute("""select max(seq)+1 from account_ssh_keys + where account_id = %s""", account_id) + seq = cur.fetchall()[0][0] + if seq is None: + seq = 1 + cur.execute("""insert into account_ssh_keys + (ssh_public_key, valid, account_id, seq) + values + (%s, 'Y', %s, %s)""", + (key.strip(), account_id, seq)) + + # account_external_ids + ## external_id + if not cur.execute("""select account_id from account_external_ids + where account_id = %s and external_id = %s""", + (account_id, user_details['openid_external_id'])): + cur.execute("""insert into account_external_ids + (account_id, email_address, external_id) + values (%s, %s, %s)""", + (account_id, user_details['email'], user_details['openid_external_id'])) + if not cur.execute("""select account_id from account_external_ids + where account_id = %s and external_id = %s""", + (account_id, "username:%s" % username)): + cur.execute("""insert into account_external_ids + (account_id, external_id) values (%s, %s)""", + (account_id, "username:%s" % username)) + + if user_details.get('email', None) is not None: + if not cur.execute("""select account_id from account_external_ids + where account_id = %s and external_id = %s""", + (account_id, "mailto:%s" % user_details['email'])): + cur.execute("""insert into account_external_ids + (account_id, email_address, external_id) + values (%s, %s, %s)""", + (account_id, user_details['email'], "mailto:%s" % + user_details['email'])) + + if account_id is not None: + # account_group_members + # user_details['add_groups'] is a list of group names for which the + # user is either "Approved" or "Administrator" + + groups_to_add = [] + groups_to_watch = {} + groups_to_rm = {} + + for group in user_details['add_groups']: + # if you are in the group nova-core, that should also put you in nova + add_groups = group_implies_groups[group_ids[group]] + add_groups.append(group_ids[group]) + for add_group in add_groups: + if add_group not in groups_to_add: + groups_to_add.append(add_group) + # We only want to add watches for direct project membership groups + groups_to_watch[group_ids[group]] = group + + # groups_to_add is now the full list of all groups we think the user + # should belong to. we want to limit the users groups to this list + for group in groups: + if group_ids[group] not in groups_to_add: + if group not in groups_to_rm.values(): + groups_to_rm[group_ids[group]] = group + + for group_id in groups_to_add: + if not cur.execute("""select account_id from account_group_members + where account_id = %s and group_id = %s""", + (account_id, group_id)): + # The current user does not exist in the group. Add it. + cur.execute("""insert into account_group_members + (account_id, group_id) + values (%s, %s)""", (account_id, group_id)) + os_project_name = groups_to_watch.get(group_id, None) + if os_project_name is not None: + if os_project_name.endswith("-core"): + os_project_name = os_project_name[:-5] + os_project_name = "openstack/%s" % os_project_name + if os_project_name in projects: + if not cur.execute("""select account_id + from account_project_watches + where account_id = %s + and project_name = %s""", + (account_id, os_project_name)): + cur.execute("""insert into account_project_watches + VALUES + ("Y", "N", "N", %s, %s, "*")""", + (account_id, os_project_name)) + + for (group_id, group_name) in groups_to_rm.items(): + cur.execute("""delete from account_group_members + where account_id = %s and group_id = %s""", + (account_id, group_id)) + +os.system("ssh -i %s -p29418 %s@localhost gerrit flush-caches" % + (GERRIT_SSH_KEY, GERRIT_USER)) + +conn.commit() diff --git a/modules/gerrit/files/scripts/update_users.py b/modules/gerrit/files/scripts/update_users.py new file mode 100644 index 0000000000..6ebf7b1afc --- /dev/null +++ b/modules/gerrit/files/scripts/update_users.py @@ -0,0 +1,430 @@ +#! /usr/bin/env python +# Copyright (C) 2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Synchronize Gerrit users from Launchpad. +# TODO items: +# 1. add a temporary (instance level) object store for the launchpad class +# 2. split out the two classes into separate files to be used as a library + +import os +import ConfigParser +import StringIO +import paramiko +import json +import logging +import uuid +from launchpadlib.launchpad import Launchpad +from launchpadlib.uris import LPNET_SERVICE_ROOT + +from datetime import datetime + +from openid.consumer import consumer +from openid.cryptutil import randomString + +GERRIT_USER = os.environ.get('GERRIT_USER', 'launchpadsync') +GERRIT_CONFIG = os.environ.get('GERRIT_CONFIG', + '/home/gerrit2/review_site/etc/gerrit.config') +GERRIT_SECURE_CONFIG = os.environ.get('GERRIT_SECURE_CONFIG', + '/home/gerrit2/review_site/etc/secure.config') +GERRIT_SSH_KEY = os.environ.get('GERRIT_SSH_KEY', + '/home/gerrit2/.ssh/launchpadsync_rsa') +GERRIT_CACHE_DIR = os.path.expanduser(os.environ.get('GERRIT_CACHE_DIR', + '~/.launchpadlib/cache')) +GERRIT_CREDENTIALS = os.path.expanduser(os.environ.get('GERRIT_CREDENTIALS', + '~/.launchpadlib/creds')) +GERRIT_BACKUP_PATH = os.environ.get('GERRIT_BACKUP_PATH', + '/home/gerrit2/dbupdates') + +logging.basicConfig(format='%(asctime)-6s: %(name)s - %(levelname)s - %(message)s', filename='/var/log/gerrit/update_users.log') +logger= logging.getLogger('update_users') +logger.setLevel(logging.INFO) + +for check_path in (os.path.dirname(GERRIT_CACHE_DIR), + os.path.dirname(GERRIT_CREDENTIALS), + GERRIT_BACKUP_PATH): + if not os.path.exists(check_path): + os.makedirs(check_path) + +def get_broken_config(filename): + """ gerrit config ini files are broken and have leading tabs """ + text = "" + with open(filename,"r") as conf: + for line in conf.readlines(): + text = "%s%s" % (text, line.lstrip()) + + fp = StringIO.StringIO(text) + c=ConfigParser.ConfigParser() + c.readfp(fp) + return c + +gerrit_config = get_broken_config(GERRIT_CONFIG) +secure_config = get_broken_config(GERRIT_SECURE_CONFIG) + +DB_USER = gerrit_config.get("database", "username") +DB_PASS = secure_config.get("database","password") +DB_DB = gerrit_config.get("database","database") + +def make_db_backup(): + db_backup_file = "%s.%s.sql" % (DB_DB, datetime.isoformat(datetime.now())) + db_backup_path = os.path.join(GERRIT_BACKUP_PATH, db_backup_file) + retval = os.system("mysqldump --opt -u%s -p%s %s > %s" % + (DB_USER, DB_PASS, DB_DB, db_backup_path)) + if retval != 0: + logger.error("Problem taking a db dump, aborting db update") + sys.exit(retval) + +class LaunchpadAction(object): + def __init__(self): + logger.info('Connecting to Launchpad') + self.launchpad= Launchpad.login_with('Gerrit User Sync', LPNET_SERVICE_ROOT, + GERRIT_CACHE_DIR, + credentials_file = GERRIT_CREDENTIALS) + + logger.info('Getting Launchpad teams') + self.lp_teams= self.get_all_sub_teams('openstack', []) + + def get_all_sub_teams(self, team, have_teams): + for sub_team in self.launchpad.people[team].sub_teams: + if sub_team.name not in have_teams: + have_teams = self.get_all_sub_teams(sub_team.name, have_teams) + have_teams.append(team) + return have_teams + + def get_sub_teams(self, team): + sub_teams= [] + for sub_team in self.launchpad.people[team].sub_teams: + sub_teams.append(sub_team.name) + return sub_teams + + def get_teams(self): + return self.lp_teams + + def get_all_users(self): + logger.info('Getting Launchpad users') + users= [] + for team in self.lp_teams: + for detail in self.launchpad.people[team].members_details: + if (detail.status == 'Approved' or detail.status == 'Administrator'): + name= detail.self_link.split('/')[-1] + if ((users.count(name) == 0) and (name not in self.lp_teams)): + users.append(name) + return users + + def get_user_data(self, user): + return self.launchpad.people[user] + + def get_team_members(self, team, gerrit): + users= [] + for detail in self.launchpad.people[team].members_details: + if (detail.status == 'Approved' or detail.status == 'Administrator'): + name= detail.self_link.split('/')[-1] + # if we found a subteam + if name in self.lp_teams: + # check subteam for implied subteams + for implied_group in gerrit.get_implied_groups(name): + if implied_group in self.lp_teams: + users.extend(self.get_team_members(implied_group, gerrit)) + users.extend(self.get_team_members(name, gerrit)) + continue + users.append(name) + # check team for implied teams + for implied_group in gerrit.get_implied_groups(team): + if implied_group in self.lp_teams: + users.extend(self.get_team_members(implied_group, gerrit)) + # filter out dupes + users= list(set(users)) + return users + + def get_team_watches(self, team): + users= [] + for detail in self.launchpad.people[team].members_details: + if (detail.status == 'Approved' or detail.status == 'Administrator'): + name= detail.self_link.split('/')[-1] + if name in self.lp_teams: + continue + if users.count(name) == 0: + users.append(name) + return users + + def get_team_display_name(self, team): + team_data = self.launchpad.people[team] + return team_data.display_name + +class GerritAction(object): + def __init__(self): + logger.info('Connecting to Gerrit') + self.ssh= paramiko.SSHClient() + self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + self.ssh.connect('localhost', username=GERRIT_USER, port=29418, key_filename=GERRIT_SSH_KEY) + + def cleanup(self): + logger.info('Closing connection to Gerrit') + self.ssh.close() + + def run_query(self, query): + command= 'gerrit gsql --format JSON -c "{0}"'.format(query) + stdin, stdout, stderr= self.ssh.exec_command(command) +# trying to get stdout return code or stderr can hang with large result sets +# for line in stderr: +# logger.error(line) + return stdout + + def get_groups(self): + logger.info('Getting Gerrit groups') + groups= [] + query= "select name from account_groups" + stdout= self.run_query(query) + for line in stdout: + row= json.loads(line) + if row['type'] == 'row': + group= row['columns']['name'] + groups.append(group) + return groups + + def get_users(self): + logger.info('Getting Gerrit users') + users= [] + query= "select external_id from account_external_ids" + stdout= self.run_query(query) + for line in stdout: + row= json.loads(line) + if row['type'] == 'row': + user= row['columns']['external_id'].replace('username:','') + users.append(user) + return users + + def get_group_id(self, group_name): + query= "select group_id from account_groups where name='{0}'".format(group_name) + stdout= self.run_query(query) + line= stdout.readline() + row= json.loads(line) + if row['type'] == 'row': + return row['columns']['group_id'] + else: + return 0 + + def get_user_id(self, user_name): + query= "select account_id from account_external_ids where external_id='username:{0}'".format(user_name) + stdout= self.run_query(query) + line= stdout.readline() + row= json.loads(line) + return row['columns']['account_id'] + + def get_users_from_group(self, group_name): + logger.info('Getting Gerrit users from group %s', group_name) + users= [] + gid= self.get_group_id(group_name) + + query= "select external_id from account_external_ids join account_group_members on account_group_members.account_id=account_external_ids.account_id where account_group_members.group_id={0} and external_id like 'username%%'".format(gid) + stdout= self.run_query(query) + for line in stdout: + row= json.loads(line) + if row['type'] == 'row': + user= row['columns']['external_id'].replace('username:','') + users.append(user) + return users + + def get_users_from_watches(self, group_name): + logger.info('Getting Gerrit users from watch list %s', group_name) + users= [] + if group_name.endswith("-core"): + group_name = group_name[:-5] + group_name = "openstack/{0}".format(group_name) + + query= "select external_id from account_external_ids join account_project_watches on account_project_watches.account_id=account_external_ids.account_id where account_project_watches.project_name like '{0}' and external_id like 'username%%'".format(group_name) + stdout= self.run_query(query) + for line in stdout: + row= json.loads(line) + if row['type'] == 'row': + user= row['columns']['external_id'].replace('username:','') + users.append(user) + return users + + + def get_implied_groups(self, group_name): + gid= self.get_group_id(group_name) + groups= [] + query= "select name from account_groups join account_group_includes on account_group_includes.include_id=account_groups.group_id where account_group_includes.group_id={0}".format(gid) + stdout= self.run_query(query) + for line in stdout: + row= json.loads(line) + if row['type'] == 'row': + group= row['columns']['name'] + groups.append(group) + return groups + + def add_group(self, group_name, group_display_name): + logger.info('New group %s (%s)', group_display_name, group) + query= "insert into account_group_id (s) values (NULL)" + stdout= self.run_query(query) + row= json.loads(stdout.readline()) + if row['rowCount'] is not 1: + print "Could not get a new account group ID" + raise + query= "select max(s) from account_group_id" + stdout= self.run_query(query) + row= json.loads(stdout.readline()) + gid= row['columns']['max(s)'] + full_uuid= "{0}{1}".format(uuid.uuid4().hex, uuid.uuid4().hex[:8]) + query= "insert into account_groups (group_id, group_type, owner_group_id, name, description, group_uuid) values ({0}, 'INTERNAL', 1, '{1}', '{2}', '{3}')". format(gid, group_name, group_display_name, full_uuid) + self.run_query(query) + query= "insert into account_group_names (group_id, name) values ({0}, '{1}')".format(gid, group_name) + self.run_query(query) + + def add_user(self, user_name, user_data): + logger.info("Adding Gerrit user %s", user_name) + openid_consumer = consumer.Consumer(dict(id=randomString(16, '0123456789abcdef')), None) + openid_request = openid_consumer.begin("https://launchpad.net/~%s" % user_data.name) + user_openid_external_id = openid_request.endpoint.getLocalID() + query= "select account_id from account_external_ids where external_id in ('{0}')".format(user_openid_external_id) + stdout= self.run_query(query) + row= json.loads(stdout.readline()) + if row['type'] == 'row': + # we have a result so this is an updated user name + account_id= row['columns']['account_id'] + query= "update account_external_ids set external_id='{0}' where external_id like 'username%%' and account_id = {1}".format('username:%s' % user_name, account_id) + self.run_query(query) + else: + # we really do have a new user + user_ssh_keys= ["%s %s %s" % ('ssh-%s' % key.keytype.lower(), key.keytext, key.comment) for key in user_data.sshkeys] + user_email= None + try: + email = user_data.preferred_email_address.email + except ValueError: + pass + query= "insert into account_id (s) values (NULL)" + self.run_query(query) + query= "select max(s) from account_id" + stdout= self.run_query(query) + row= json.loads(stdout.readline()) + uid= row['columns']['max(s)'] + query= "insert into accounts (account_id, full_name, preferred_email) values ({0}, '{1}', '{2}')".format(uid, user_name, user_email) + self.run_query(query) + keyno= 1 + for key in user_ssh_keys: + query= "insert into account_ssh_keys (ssh_public_key, valid, account_id, seq) values ('{0}', 'Y', {1}, {2})".format(key.strip(), uid, keyno) + self.run_query(query) + keyno = keyno + 1 + query= "insert into account_external_ids (account_id, email_address, external_id) values ({0}, '{1}', '{2}')".format(uid, user_email, user_openid_external_id) + self.run_query(query) + query= "insert into account_external_ids (account_id, external_id) values ({0}, '{1}')".format(uid, "username:%s" % user_name) + self.run_query(query) + if user_email is not None: + query= "insert into account_external_ids (account_id, email_address, external_id) values ({0}. '{1}', '{2}')".format(uid, user_email, "mailto:%s" % user_email) + return None + + def add_user_to_group(self, user_name, group_name): + logger.info("Adding Gerrit user %s to group %s", user_name, group_name) + uid= self.get_user_id(user_name) + gid= self.get_group_id(group_name) + if gid is 0: + print "Trying to add user {0} to non-existent group {1}".format(user_name, group_name) + raise + query= "insert into account_group_members (account_id, group_id) values ({0}, {1})".format(uid, gid) + self.run_query(query) + + def add_user_to_watch(self, user_name, group_name): + logger.info("Adding Gerrit user %s to watch group %s", user_name, group_name) + uid= self.get_user_id(user_name) + if group_name.endswith("-core"): + group_name = group_name[:-5] + group_name = "openstack/{0}".format(group_name) + query= "insert into account_project_watches VALUES ('Y', 'N', 'N', {0}, '{1}', '*')". format(uid, group_name) + self.run_query(query) + + + def del_user_from_group(self, user_name, group_name): + logger.info("Deleting Gerrit user %s from group %s", user_name, group_name) + uid= self.get_user_id(user_name) + gid= self.get_group_id(group_name) + query= "delete from account_group_members where account_id = {0} and group_id = {1}".format(uid, gid) + self.run_query(query) + if group_name.endswith("-core"): + group_name = group_name[:-5] + group_name= "openstack/{0}".format(group_name) + query= "delete from account_project_watches where account_id = {0} and project_name= '{1}'".format(uid, group_name) + self.run_query(query) + + def rebuild_sub_groups(self, group, sub_groups): + gid= self.get_group_id(group) + for sub_group in sub_groups: + sgid= self.get_group_id(sub_group) + query= "select group_id from account_group_includes where group_id={0} and include_id={1}".format(gid, sgid) + stdout= self.run_query(query) + row= json.loads(stdout.readline()) + if row['type'] != 'row': + logger.info('Adding implied group %s to group %s', group, sub_group) + query= "insert into account_group_includes (group_id, include_id) values ({0}, {1})".format(gid, sgid) + self.run_query(query) + + +# Actual work starts here! + +lp= LaunchpadAction() +gerrit= GerritAction() + +logger.info('Making DB backup') +make_db_backup() + +logger.info('Starting group reconcile') +lp_groups= lp.get_teams() +gerrit_groups= gerrit.get_groups() + +group_diff= filter(lambda a: a not in gerrit_groups, lp_groups) +for group in group_diff: + group_display_name= lp.get_team_display_name(group) + gerrit.add_group(group, group_display_name) + +for group in lp_groups: + sub_group= lp.get_sub_teams(group) + if sub_group: + gerrit.rebuild_sub_groups(group, sub_group) + +logger.info('End group reconcile') + +logger.info('Starting user reconcile') +lp_users= lp.get_all_users() +gerrit_users= gerrit.get_users() + +user_diff= filter(lambda a: a not in gerrit_users, lp_users) +for user in user_diff: + gerrit.add_user(user, lp.get_user_data(user)) + +logger.info('End user reconcile') + +logger.info('Starting user to group reconcile') +lp_groups= lp.get_teams() +for group in lp_groups: + # First find users to attach to groups + gerrit_group_users= gerrit.get_users_from_group(group) + lp_group_users= lp.get_team_members(group, gerrit) + + group_diff= filter(lambda a: a not in gerrit_group_users, lp_group_users) + for user in group_diff: + gerrit.add_user_to_group(user, group) + # Second find users to attach to watches + lp_group_watches= lp.get_team_watches(group) + gerrit_group_watches= gerrit.get_users_from_watches(group) + group_diff= filter(lambda a: a not in gerrit_group_watches, lp_group_watches) + for user in group_diff: + gerrit.add_user_to_watch(user, group) + # Third find users to remove from groups/watches + group_diff= filter(lambda a: a not in lp_group_users, gerrit_group_users) + for user in group_diff: + gerrit.del_user_from_group(user, group) + +logger.info('Ending user to group reconcile') + +gerrit.cleanup() diff --git a/modules/gerrit/manifests/init.pp b/modules/gerrit/manifests/init.pp index 0e3bbb8ab1..e1daec7346 100644 --- a/modules/gerrit/manifests/init.pp +++ b/modules/gerrit/manifests/init.pp @@ -109,29 +109,27 @@ class gerrit($virtual_hostname='', # Skip cron jobs if we're in test mode if ($testmode == false) { - cron { "gerritupdateci": - user => gerrit2, - minute => "*/15", - command => 'sleep $((RANDOM\%60)) && cd /home/gerrit2/openstack-ci && /usr/bin/git pull -q origin master' - } cron { "gerritsyncusers": user => gerrit2, minute => "*/15", - command => 'sleep $((RANDOM\%60+60)) && cd /home/gerrit2/openstack-ci && python gerrit/update_gerrit_users.py' + command => 'sleep $((RANDOM\%60+60)) && python /usr/local/gerrit/scripts/update_gerrit_users.py', + require => File['/usr/local/gerrit/scripts'], } cron { "gerritclosepull": user => gerrit2, minute => "*/5", - command => 'sleep $((RANDOM\%60+90)) && cd /home/gerrit2/openstack-ci && python gerrit/close_pull_requests.py' + command => 'sleep $((RANDOM\%60+90)) && python /usr/local/gerrit/scripts/close_pull_requests.py', + require => File['/usr/local/gerrit/scripts'], } cron { "expireoldreviews": user => gerrit2, hour => 6, minute => 3, - command => 'cd /home/gerrit2/openstack-ci && python gerrit/expire_old_reviews.py' + command => 'python /usr/local/gerrit/scripts/expire_old_reviews.py', + require => File['/usr/local/gerrit/scripts'], } cron { "gerrit_repack": @@ -494,4 +492,23 @@ class gerrit($virtual_hostname='', require => File['/etc/init.d/gerrit'], refreshonly => true, } + + file { '/usr/local/gerrit': + owner => 'root', + group => 'root', + mode => 755, + ensure => 'directory', + } + + file { '/usr/local/gerrit/scripts': + owner => 'root', + group => 'root', + mode => 755, + ensure => 'directory', + recurse => true, + require => File['/usr/local/gerrit'], + source => [ + "puppet:///modules/gerrit/scripts", + ], + } }