Move gerrit scripts to puppet.

Take the things from openstack-ci/gerrit and move them directly
in to the puppet module. Install them using the model we're using
for the jenkins slave scripts.

Change-Id: I420b2b895bd57d40232b2cdda437617373a82890

modules/gerrit/files/change-merged

@@ -1,3 +1,3 @@
 #!/bin/sh
 
-python ~/openstack-ci/gerrit/update_bug.py change-merged "$@"
+python /usr/local/gerrit/scripts/update_bug.py change-merged "$@"

modules/gerrit/files/patchset-created

@@ -1,4 +1,4 @@
 #!/bin/sh
 
-python ~/openstack-ci/gerrit/update_blueprint.py patchset-created "$@"
-python ~/openstack-ci/gerrit/update_bug.py patchset-created "$@"
+python /usr/local/gerrit/scripts/update_blueprint.py patchset-created "$@"
+python /usr/local/gerrit/scripts/update_bug.py patchset-created "$@"

modules/gerrit/files/scripts/close_pull_requests.py

@@ -0,0 +1,78 @@
+#! /usr/bin/env python
+# Copyright (C) 2011 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# Close Github pull requests with instructions to use Gerrit for
+# code review.  The list of projects is found in github.config
+# and should look like:
+
+# [project "GITHUB_PROJECT"]
+# close_pull = true
+
+# Github authentication information is read from github.secure.config,
+# which should look like:
+
+# [github]
+# username = GITHUB_USERNAME
+# api_token = GITHUB_API_TOKEN
+
+import github2.client
+import os
+import StringIO
+import ConfigParser
+import logging
+import re
+
+logging.basicConfig(level=logging.ERROR)
+
+GITHUB_CONFIG = os.environ.get('GITHUB_CONFIG',
+                               '/home/gerrit2/github.config')
+GITHUB_SECURE_CONFIG = os.environ.get('GITHUB_SECURE_CONFIG',
+                                      '/home/gerrit2/github.secure.config')
+
+MESSAGE = """Thank you for contributing to OpenStack!
+
+%(project)s uses Gerrit for code review.
+
+Please visit http://wiki.openstack.org/GerritWorkflow and follow the instructions there to upload your change to Gerrit.
+"""
+
+PROJECT_RE = re.compile(r'^project\s+"(.*)"$')
+
+secure_config = ConfigParser.ConfigParser()
+secure_config.read(GITHUB_SECURE_CONFIG)
+config = ConfigParser.ConfigParser()
+config.read(GITHUB_CONFIG)
+
+github = github2.client.Github(requests_per_second=1.0,
+                               username=secure_config.get("github", "username"),
+                               api_token=secure_config.get("github", "api_token"))
+
+for section in config.sections():
+    # Each section looks like [project "openstack/project"]
+    m = PROJECT_RE.match(section)
+    if not m: continue
+    project = m.group(1)
+
+    # Make sure we're supposed to close pull requests for this project:
+    if not (config.has_option(section, "close_pull") and
+            config.get(section, "close_pull").lower() == 'true'):
+        continue
+
+    # Close each pull request
+    pull_requests = github.pull_requests.list(project)
+    for req in pull_requests:
+        vars = dict(project=project)
+        github.issues.comment(project, req.number, MESSAGE%vars)
+        github.issues.close(project, req.number)

modules/gerrit/files/scripts/expire_old_reviews.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+# Copyright (c) 2012 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This script is designed to expire old code reviews that have not been touched
+# using the following rules:
+# 1. if open and no activity in 2 weeks, expire
+# 2. if negative comment and no activity in 1 week, expire
+
+import os
+import paramiko
+import json
+import logging
+
+GERRIT_USER = os.environ.get('GERRIT_USER', 'launchpadsync')
+GERRIT_SSH_KEY = os.environ.get('GERRIT_SSH_KEY',
+                                 '/home/gerrit2/.ssh/launchpadsync_rsa')
+
+logging.basicConfig(format='%(asctime)-6s: %(name)s - %(levelname)s - %(message)s', filename='/var/log/gerrit/expire_reviews.log')
+logger= logging.getLogger('expire_reviews')
+logger.setLevel(logging.INFO)
+
+logger.info('Starting expire reviews')
+logger.info('Connecting to Gerrit')
+
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ssh.connect('localhost', username=GERRIT_USER, key_filename=GERRIT_SSH_KEY, port=29418)
+
+def expire_patch_set(patch_id, patch_subject, has_negative):
+  if has_negative:
+    message= 'code review expired after 1 week of no activity after a negative review'
+  else:
+    message= 'code review expired after 2 weeks of no activity'
+  command='gerrit review --abandon --message="{0}" {1}'.format(message, patch_id)
+  logger.info('Expiring: %s - %s: %s', patch_id, patch_subject, message)
+  stdin, stdout, stderr = ssh.exec_command(command)
+  if stdout.channel.recv_exit_status() != 0:
+    logger.error(stderr.read())
+
+# Query all open with no activity for 2 weeks
+logger.info('Searching no activity for 2 weeks')
+stdin, stdout, stderr = ssh.exec_command('gerrit query --current-patch-set --format JSON status:open age:2w')
+
+for line in stdout:
+  row= json.loads(line)
+  if not row.has_key('rowCount'):
+    expire_patch_set(row['currentPatchSet']['revision'], row['subject'], False)
+
+# Query all reviewed with no activity for 1 week
+logger.info('Searching no activity on negative review for 1 week')
+stdin, stdout, stderr = ssh.exec_command('gerrit query --current-patch-set --all-approvals --format JSON status:reviewed age:1w')
+
+for line in stdout:
+  row= json.loads(line)
+  if not row.has_key('rowCount'):
+    # Search for negative approvals
+    for approval in row['currentPatchSet']['approvals']:
+      if approval['value'] == '-1':
+        expire_patch_set(row['currentPatchSet']['revision'], row['subject'], True)
+        break
+
+logger.info('End expire review')

modules/gerrit/files/scripts/get_group_uuid.py

@@ -0,0 +1,29 @@
+import argparse
+import paramiko
+import json
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--host", dest="host", default="review.openstack.org",
+                    help="gerrit host to connect to")
+parser.add_argument("--port", dest="port", action='store', type=int,
+                    default=29418, help="gerrit port to connect to")
+parser.add_argument("groups", nargs=1)
+
+options = parser.parse_args()
+
+
+client = paramiko.SSHClient()
+client.load_system_host_keys()
+client.set_missing_host_key_policy(paramiko.WarningPolicy())
+client.connect(options.host, port=options.port)
+
+group = options.groups[0]
+query = "select group_uuid from account_groups where name = '%s'" % group
+command = 'gerrit gsql --format JSON -c "%s"' % query
+stdin, stdout, stderr = client.exec_command(command)
+
+for line in stdout:
+    row = json.loads(line)
+    if row['type'] == 'row':
+        print row['columns']['group_uuid']
+    ret = stdout.channel.recv_exit_status()

modules/gerrit/files/scripts/update_blueprint.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python
+# Copyright (c) 2011 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This is designed to be called by a gerrit hook.  It searched new
+# patchsets for strings like "blueprint FOO" or "bp FOO" and updates
+# corresponding Launchpad blueprints with links back to the change.
+
+from launchpadlib.launchpad import Launchpad
+from launchpadlib.uris import LPNET_SERVICE_ROOT
+import os
+import argparse
+import re
+import subprocess
+
+import StringIO
+import ConfigParser
+import MySQLdb
+
+BASE_DIR = '/home/gerrit2/review_site'
+GERRIT_CACHE_DIR = os.path.expanduser(os.environ.get('GERRIT_CACHE_DIR',
+                                '~/.launchpadlib/cache'))
+GERRIT_CREDENTIALS = os.path.expanduser(os.environ.get('GERRIT_CREDENTIALS',
+                                '~/.launchpadlib/creds'))
+GERRIT_CONFIG = os.environ.get('GERRIT_CONFIG',
+                                 '/home/gerrit2/review_site/etc/gerrit.config')
+GERRIT_SECURE_CONFIG = os.environ.get('GERRIT_SECURE_CONFIG',
+                                 '/home/gerrit2/review_site/etc/secure.config')
+SPEC_RE = re.compile(r'(blueprint|bp)\s*[#:]?\s*(\S+)', re.I)
+BODY_RE = re.compile(r'^\s+.*$')
+
+def get_broken_config(filename):
+    """ gerrit config ini files are broken and have leading tabs """
+    text = ""
+    with open(filename,"r") as conf:
+        for line in conf.readlines():
+            text = "%s%s" % (text, line.lstrip())
+
+    fp = StringIO.StringIO(text)
+    c=ConfigParser.ConfigParser()
+    c.readfp(fp)
+    return c
+
+GERRIT_CONFIG = get_broken_config(GERRIT_CONFIG)
+SECURE_CONFIG = get_broken_config(GERRIT_SECURE_CONFIG)
+DB_USER = GERRIT_CONFIG.get("database", "username")
+DB_PASS = SECURE_CONFIG.get("database","password")
+DB_DB = GERRIT_CONFIG.get("database","database")
+
+def update_spec(launchpad, project, name, subject, link, topic=None):
+    # For testing, if a project doesn't match openstack/foo, use
+    # the openstack-ci project instead.
+    group, project = project.split('/')
+    if group != 'openstack':
+        project = 'openstack-ci'
+
+    spec = launchpad.projects[project].getSpecification(name=name)
+    if not spec: return
+
+    if spec.whiteboard:
+        wb = spec.whiteboard.strip()
+    else:
+        wb = ''
+    changed = False
+    if topic:
+        topiclink = '%s/#q,topic:%s,n,z' % (link[:link.find('/',8)],
+                                            topic)
+        if topiclink not in wb:
+            wb += "\n\n\nGerrit topic: %(link)s" % dict(link=topiclink)
+            changed = True
+
+    if link not in wb:
+        wb += "\n\n\nAddressed by: %(link)s\n    %(subject)s\n" % dict(subject=subject,
+                                                                      link=link)
+        changed = True
+
+    if changed:
+        spec.whiteboard = wb
+        spec.lp_save()
+
+def find_specs(launchpad, dbconn, args):
+    git_log = subprocess.Popen(['git',
+                                '--git-dir=' + BASE_DIR + '/git/' + args.project + '.git',
+                                'log', '--no-merges',
+                                args.commit + '^1..' + args.commit],
+                               stdout=subprocess.PIPE).communicate()[0]
+
+    cur = dbconn.cursor()
+    cur.execute("select subject, topic from changes where change_key=%s", args.change)
+    subject, topic = cur.fetchone()
+    specs = set([m.group(2) for m in SPEC_RE.finditer(git_log)])
+
+    if topic:
+        topicspec = topic.split('/')[-1]
+        specs |= set([topicspec])
+
+    for spec in specs:
+        update_spec(launchpad, args.project, spec, subject,
+                    args.change_url, topic)
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('hook')
+    #common
+    parser.add_argument('--change', default=None)
+    parser.add_argument('--change-url', default=None)
+    parser.add_argument('--project', default=None)
+    parser.add_argument('--branch', default=None)
+    parser.add_argument('--commit', default=None)
+    #change-merged
+    parser.add_argument('--submitter', default=None)
+    # patchset-created
+    parser.add_argument('--uploader', default=None)
+    parser.add_argument('--patchset', default=None)
+
+    args = parser.parse_args()
+
+    launchpad = Launchpad.login_with('Gerrit User Sync', LPNET_SERVICE_ROOT,
+                                     GERRIT_CACHE_DIR,
+                                     credentials_file = GERRIT_CREDENTIALS,
+                                     version='devel')
+
+    conn = MySQLdb.connect(user = DB_USER, passwd = DB_PASS, db = DB_DB)
+
+    find_specs(launchpad, conn, args)
+
+if __name__ == '__main__':
+    main()

modules/gerrit/files/scripts/update_bug.py

@@ -0,0 +1,200 @@
+#!/usr/bin/env python
+# Copyright (c) 2011 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This is designed to be called by a gerrit hook.  It searched new
+# patchsets for strings like "bug FOO" and updates corresponding Launchpad
+# bugs status.
+
+from launchpadlib.launchpad import Launchpad
+from launchpadlib.uris import LPNET_SERVICE_ROOT
+import os
+import argparse
+import re
+import subprocess
+
+
+BASE_DIR = '/home/gerrit2/review_site'
+GERRIT_CACHE_DIR = os.path.expanduser(os.environ.get('GERRIT_CACHE_DIR',
+                                '~/.launchpadlib/cache'))
+GERRIT_CREDENTIALS = os.path.expanduser(os.environ.get('GERRIT_CREDENTIALS',
+                                '~/.launchpadlib/creds'))
+
+
+def add_change_proposed_message(bugtask, change_url, project, branch):
+    subject = 'Fix proposed to %s (%s)' % (short_project(project), branch)
+    body = 'Fix proposed to branch: %s\nReview: %s' % (branch, change_url)
+    bugtask.bug.newMessage(subject=subject, content=body)
+
+
+def add_change_merged_message(bugtask, change_url, project, commit,
+                              submitter, branch, git_log):
+    subject = 'Fix merged to %s (%s)' % (short_project(project), branch)
+    git_url = 'http://github.com/%s/commit/%s' % (project, commit)
+    body = '''Reviewed:  %s
+Committed: %s
+Submitter: %s
+Branch:    %s\n''' % (change_url, git_url, submitter, branch)
+    body = body + '\n' + git_log
+    bugtask.bug.newMessage(subject=subject, content=body)
+
+
+def set_in_progress(bugtask, launchpad, uploader, change_url):
+    """Set bug In progress with assignee being the uploader"""
+
+    # Retrieve uploader from Launchpad. Use email as search key if
+    # provided, and only set if there is a clear match.
+    try:
+        searchkey = uploader[uploader.rindex("(") + 1:-1]
+    except ValueError:
+        searchkey = uploader
+    persons = launchpad.people.findPerson(text=searchkey)
+    if len(persons) == 1:
+        bugtask.assignee = persons[0]
+
+    bugtask.status = "In Progress"
+    bugtask.lp_save()
+
+
+def set_fix_committed(bugtask):
+    """Set bug fix committed"""
+
+    bugtask.status = "Fix Committed"
+    bugtask.lp_save()
+
+
+def release_fixcommitted(bugtask):
+    """Set bug FixReleased if it was FixCommitted"""
+
+    if bugtask.status == u'Fix Committed':
+        bugtask.status = "Fix Released"
+        bugtask.lp_save()
+
+
+def tag_in_branchname(bugtask, branch):
+    """Tag bug with in-branch-name tag (if name is appropriate)"""
+
+    lp_bug = bugtask.bug
+    branch_name = branch.replace('/', '-')
+    if branch_name.replace('-', '').isalnum():
+        lp_bug.tags = lp_bug.tags + ["in-%s" % branch_name]
+        lp_bug.tags.append("in-%s" % branch_name)
+        lp_bug.lp_save()
+
+
+def short_project(full_project_name):
+    """Return the project part of the git repository name"""
+    return full_project_name.split('/')[-1]
+
+
+def git2lp(full_project_name):
+    """Convert Git repo name to Launchpad project"""
+    project_map = {
+        'openstack/python-glanceclient': 'glance',
+        'openstack/python-keystoneclient': 'keystone',
+        'openstack/python-melangeclient': 'melange',
+        'openstack/python-novaclient': 'nova',
+        'openstack/python-quantumclient': 'quantum',
+        'openstack/openstack-ci-puppet': 'openstack-ci',
+        'openstack-ci/devstack-gate': 'openstack-ci',
+        }
+    return project_map.get(full_project_name, short_project(full_project_name))
+
+
+def process_bugtask(launchpad, bugtask, git_log, args):
+    """Apply changes to bugtask, based on hook / branch..."""
+
+    if args.hook == "change-merged":
+        if args.branch == 'master':
+            set_fix_committed(bugtask)
+        elif args.branch == 'milestone-proposed':
+            release_fixcommitted(bugtask)
+        else:
+            tag_in_branchname(bugtask, args.branch)
+        add_change_merged_message(bugtask, args.change_url, args.project,
+                                  args.commit, args.submitter, args.branch,
+                                  git_log)
+
+    if args.hook == "patchset-created":
+        if args.branch == 'master':
+            set_in_progress(bugtask, launchpad, args.uploader, args.change_url)
+        if args.patchset == '1':
+            add_change_proposed_message(bugtask, args.change_url,
+                                        args.project, args.branch)
+
+
+def find_bugs(launchpad, git_log, args):
+    """Find bugs referenced in the git log and return related bugtasks"""
+
+    bug_regexp = r'([Bb]ug|[Ll][Pp])[\s#:]*(\d+)'
+    tokens = re.split(bug_regexp, git_log)
+
+    # Extract unique bug tasks
+    bugtasks = {}
+    for token in tokens:
+        if re.match('^\d+$', token) and (token not in bugtasks):
+            try:
+                lp_bug = launchpad.bugs[token]
+                for lp_task in lp_bug.bug_tasks:
+                    if lp_task.bug_target_name == git2lp(args.project):
+                        bugtasks[token] = lp_task
+                        break
+            except KeyError:
+                # Unknown bug
+                pass
+
+    return bugtasks.values()
+
+
+def extract_git_log(args):
+    """Extract git log of all merged commits"""
+    cmd = ['git',
+           '--git-dir=' + BASE_DIR + '/git/' + args.project + '.git',
+           'log', '--no-merges', args.commit + '^1..' + args.commit]
+    return subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('hook')
+    #common
+    parser.add_argument('--change', default=None)
+    parser.add_argument('--change-url', default=None)
+    parser.add_argument('--project', default=None)
+    parser.add_argument('--branch', default=None)
+    parser.add_argument('--commit', default=None)
+    #change-merged
+    parser.add_argument('--submitter', default=None)
+    #patchset-created
+    parser.add_argument('--uploader', default=None)
+    parser.add_argument('--patchset', default=None)
+
+    args = parser.parse_args()
+
+    # Connect to Launchpad
+    launchpad = Launchpad.login_with('Gerrit User Sync', LPNET_SERVICE_ROOT,
+                                     GERRIT_CACHE_DIR,
+                                     credentials_file=GERRIT_CREDENTIALS,
+                                     version='devel')
+
+    # Get git log
+    git_log = extract_git_log(args)
+
+    # Process bugtasks found in git log
+    for bugtask in find_bugs(launchpad, git_log, args):
+        process_bugtask(launchpad, bugtask, git_log, args)
+
+
+if __name__ == '__main__':
+    main()

modules/gerrit/files/scripts/update_cla_group.py

@@ -0,0 +1,77 @@
+#! /usr/bin/env python
+# Copyright (C) 2011 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# Add launchpad ids listed in the wiki CLA page to the CLA group in LP.
+
+import os
+import sys
+import uuid
+import os
+import urllib
+import re
+
+import StringIO
+import ConfigParser
+
+from launchpadlib.launchpad import Launchpad
+from launchpadlib.uris import LPNET_SERVICE_ROOT
+
+DEBUG = False
+
+LP_CACHE_DIR = '~/.launchpadlib/cache'
+LP_CREDENTIALS = '~/.launchpadlib/creds'
+CONTRIBUTOR_RE = re.compile(r'.*?\|\|\s*(?P<name>.*?)\s*\|\|\s*(?P<login>.*?)\s*\|\|\s*(?P<trans>.*?)\s*\|\|.*?')
+LINK_RE = re.compile(r'\[\[.*\|\s*(?P<name>.*)\s*\]\]')
+
+for check_path in (os.path.dirname(LP_CACHE_DIR),
+                   os.path.dirname(LP_CREDENTIALS)):
+    if not os.path.exists(check_path):
+        os.makedirs(check_path)
+
+wiki_members = []
+for line in urllib.urlopen('http://wiki.openstack.org/Contributors?action=raw'):
+    m = CONTRIBUTOR_RE.match(line)
+    if m and m.group('login') and m.group('trans'):
+        login = m.group('login')
+        if login=="<#c0c0c0>'''Launchpad ID'''": continue
+        l = LINK_RE.match(login)
+        if l:
+            login = l.group('name')
+        wiki_members.append(login)
+
+launchpad = Launchpad.login_with('CLA Team Sync', LPNET_SERVICE_ROOT,
+                                 LP_CACHE_DIR,
+                                 credentials_file = LP_CREDENTIALS)
+
+lp_members = []
+
+team = launchpad.people['openstack-cla']
+for detail in team.members_details:
+    user = None
+    # detail.self_link ==
+    # 'https://api.launchpad.net/1.0/~team/+member/${username}'
+    login = detail.self_link.split('/')[-1]
+    status = detail.status
+    lp_members.append(login)
+
+for wm in wiki_members:
+    if wm not in lp_members:
+        print "Need to add %s to LP" % (wm)
+        try:
+            person = launchpad.people[wm]
+        except:
+            print 'Unable to find %s on LP'%wm
+            continue
+        status = team.addMember(person=person, status="Approved")

modules/gerrit/files/scripts/update_gerrit_users.py

@@ -0,0 +1,391 @@
+#! /usr/bin/env python
+# Copyright (C) 2011 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# Synchronize Gerrit users from Launchpad.
+
+import os
+import sys
+import uuid
+import os
+import subprocess
+
+from datetime import datetime
+
+import StringIO
+import ConfigParser
+
+import MySQLdb
+
+from launchpadlib.launchpad import Launchpad
+from launchpadlib.uris import LPNET_SERVICE_ROOT
+
+from openid.consumer import consumer
+from openid.cryptutil import randomString
+
+DEBUG = False
+
+GERRIT_USER = os.environ.get('GERRIT_USER', 'launchpadsync')
+GERRIT_CONFIG = os.environ.get('GERRIT_CONFIG',
+                                 '/home/gerrit2/review_site/etc/gerrit.config')
+GERRIT_SECURE_CONFIG = os.environ.get('GERRIT_SECURE_CONFIG',
+                                 '/home/gerrit2/review_site/etc/secure.config')
+GERRIT_SSH_KEY = os.environ.get('GERRIT_SSH_KEY',
+                                 '/home/gerrit2/.ssh/launchpadsync_rsa')
+GERRIT_CACHE_DIR = os.path.expanduser(os.environ.get('GERRIT_CACHE_DIR',
+                                '~/.launchpadlib/cache'))
+GERRIT_CREDENTIALS = os.path.expanduser(os.environ.get('GERRIT_CREDENTIALS',
+                                '~/.launchpadlib/creds'))
+GERRIT_BACKUP_PATH = os.environ.get('GERRIT_BACKUP_PATH',
+                                '/home/gerrit2/dbupdates')
+
+for check_path in (os.path.dirname(GERRIT_CACHE_DIR),
+                   os.path.dirname(GERRIT_CREDENTIALS),
+                   GERRIT_BACKUP_PATH):
+  if not os.path.exists(check_path):
+    os.makedirs(check_path)
+
+def get_broken_config(filename):
+  """ gerrit config ini files are broken and have leading tabs """
+  text = ""
+  with open(filename,"r") as conf:
+    for line in conf.readlines():
+      text = "%s%s" % (text, line.lstrip())
+
+  fp = StringIO.StringIO(text)
+  c=ConfigParser.ConfigParser()
+  c.readfp(fp)
+  return c
+
+def get_type(in_type):
+  if in_type == "RSA":
+    return "ssh-rsa"
+  else:
+    return "ssh-dsa"
+
+gerrit_config = get_broken_config(GERRIT_CONFIG)
+secure_config = get_broken_config(GERRIT_SECURE_CONFIG)
+
+DB_USER = gerrit_config.get("database", "username")
+DB_PASS = secure_config.get("database","password")
+DB_DB = gerrit_config.get("database","database")
+
+db_backup_file = "%s.%s.sql" % (DB_DB, datetime.isoformat(datetime.now()))
+db_backup_path = os.path.join(GERRIT_BACKUP_PATH, db_backup_file)
+retval = os.system("mysqldump --opt -u%s -p%s %s > %s" %
+                     (DB_USER, DB_PASS, DB_DB, db_backup_path))
+if retval != 0:
+  print "Problem taking a db dump, aborting db update"
+  sys.exit(retval)
+
+conn = MySQLdb.connect(user = DB_USER, passwd = DB_PASS, db = DB_DB)
+cur = conn.cursor()
+
+
+launchpad = Launchpad.login_with('Gerrit User Sync', LPNET_SERVICE_ROOT,
+                                 GERRIT_CACHE_DIR,
+                                 credentials_file = GERRIT_CREDENTIALS)
+
+def get_sub_teams(team, have_teams):
+    for sub_team in launchpad.people[team].sub_teams:
+        if sub_team.name not in have_teams:
+           have_teams = get_sub_teams(sub_team.name, have_teams)
+    have_teams.append(team)
+    return have_teams
+
+
+teams_todo = get_sub_teams('openstack', [])
+
+users={}
+groups={}
+groups_in_groups={}
+group_implies_groups={}
+group_ids={}
+projects = subprocess.check_output(['/usr/bin/ssh', '-p', '29418',
+    '-i', GERRIT_SSH_KEY,
+    '-l', GERRIT_USER, 'localhost',
+    'gerrit', 'ls-projects']).split('\n')
+
+for team_todo in teams_todo:
+
+  team = launchpad.people[team_todo]
+  groups[team.name] = team.display_name
+
+  # Attempt to get nested group memberships. ~nova-core, for instance, is a
+  # member of ~nova, so membership in ~nova-core should imply membership in
+  # ~nova
+  group_in_group = groups_in_groups.get(team.name, {})
+  for subgroup in team.sub_teams:
+    group_in_group[subgroup.name] = 1
+  # We should now have a dictionary of the form {'nova': {'nova-core': 1}}
+  groups_in_groups[team.name] = group_in_group
+
+  for detail in team.members_details:
+
+    user = None
+
+    # detail.self_link ==
+    # 'https://api.launchpad.net/1.0/~team/+member/${username}'
+    login = detail.self_link.split('/')[-1]
+
+    if users.has_key(login):
+      user = users[login]
+    else:
+
+      user = dict(add_groups=[])
+
+    status = detail.status
+    if (status == "Approved" or status == "Administrator"):
+      user['add_groups'].append(team.name)
+    users[login] = user
+
+# If we picked up subgroups that were not in our original list of groups
+# make sure they get added
+for (supergroup, subgroups) in groups_in_groups.items():
+  for group in subgroups.keys():
+    if group not in groups.keys():
+      groups[group] = None
+
+# account_groups
+# groups is a dict of team name to team display name
+# here, for every group we have in that dict, we're building another dict of
+# group_name to group_id - and if the database doesn't already have the
+# group, we're adding it
+for (group_name, group_display_name) in groups.items():
+  if cur.execute("select group_id from account_groups where name = %s",
+                 group_name):
+    group_ids[group_name] = cur.fetchall()[0][0]
+  else:
+    cur.execute("""insert into account_group_id (s) values (NULL)""");
+    cur.execute("select max(s) from account_group_id")
+    group_id = cur.fetchall()[0][0]
+
+    # Match the 40-char 'uuid' that java is producing
+    group_uuid = uuid.uuid4()
+    second_uuid = uuid.uuid4()
+    full_uuid = "%s%s" % (group_uuid.hex, second_uuid.hex[:8])
+
+    cur.execute("""insert into account_groups
+                   (group_id, group_type, owner_group_id,
+                    name, description, group_uuid)
+                   values
+                   (%s, 'INTERNAL', 1, %s, %s, %s)""",
+                (group_id, group_name, group_display_name, full_uuid))
+    cur.execute("""insert into account_group_names (group_id, name) values
+    (%s, %s)""",
+    (group_id, group_name))
+
+    group_ids[group_name] = group_id
+
+# account_group_includes
+# groups_in_groups should be a dict of dicts, where the key is the larger
+# group and the inner dict is a list of groups that are members of the
+# larger group. So {'nova': {'nova-core': 1}}
+for (group_name, subgroups) in groups_in_groups.items():
+  for subgroup_name in subgroups.keys():
+    try:
+      cur.execute("""insert into account_group_includes
+                       (group_id, include_id)
+                      values (%s, %s)""",
+                  (group_ids[group_name], group_ids[subgroup_name]))
+    except MySQLdb.IntegrityError:
+      pass
+
+# Make a list of implied group membership
+# building a list which is the opposite of groups_in_group. Here
+# group_implies_groups is a dict keyed by group_id containing a list of
+# group_ids of implied membership. SO: if nova is 1 and nova-core is 2:
+# {'2': [1]}
+for group_id in group_ids.values():
+    total_groups = []
+    groups_todo = [group_id]
+    while len(groups_todo) > 0:
+        current_group = groups_todo.pop()
+        total_groups.append(current_group)
+        cur.execute("""select group_id from account_group_includes
+                        where include_id = %s""", (current_group))
+        for row in cur.fetchall():
+            if row[0] != 1 and row[0] not in total_groups:
+                groups_todo.append(row[0])
+    group_implies_groups[group_id] = total_groups
+
+if DEBUG:
+    def get_group_name(in_group_id):
+      for (group_name, group_id) in group_ids.items():
+        if group_id == in_group_id:
+          return group_name
+
+    print "groups in groups"
+    for (k,v) in groups_in_groups.items():
+      print k, v
+
+    print "group_imples_groups"
+    for (k, v) in group_implies_groups.items():
+      print get_group_name(k)
+      new_groups=[]
+      for val in v:
+        new_groups.append(get_group_name(val))
+      print "\t", new_groups
+
+for (username, user_details) in users.items():
+
+  # accounts
+  account_id = None
+  if cur.execute("""select account_id from account_external_ids where
+    external_id in (%s)""", ("username:%s" % username)):
+    account_id = cur.fetchall()[0][0]
+    # We have this bad boy - all we need to do is update his group membership
+
+  else:
+
+    # We need details
+    member = launchpad.people[username]
+    if not member.is_team:
+
+      openid_consumer = consumer.Consumer(dict(id=randomString(16, '0123456789abcdef')), None)
+      openid_request = openid_consumer.begin("https://launchpad.net/~%s" % member.name)
+      user_details['openid_external_id'] = openid_request.endpoint.getLocalID()
+
+      # Handle username change
+      if cur.execute("""select account_id from account_external_ids where
+        external_id in (%s)""", user_details['openid_external_id']):
+        account_id = cur.fetchall()[0][0]
+        cur.execute("""update account_external_ids
+                          set external_id=%s
+                        where external_id like 'username%%'
+                          and account_id = %s""",
+                     ('username:%s' % username, account_id))
+      else:
+        user_details['ssh_keys'] = ["%s %s %s" % (get_type(key.keytype), key.keytext, key.comment) for key in member.sshkeys]
+
+
+        email = None
+        try:
+          email = member.preferred_email_address.email
+        except ValueError:
+          pass
+        user_details['email'] = email
+
+
+        cur.execute("""insert into account_id (s) values (NULL)""");
+        cur.execute("select max(s) from account_id")
+        account_id = cur.fetchall()[0][0]
+
+        cur.execute("""insert into accounts (account_id, full_name, preferred_email) values
+        (%s, %s, %s)""", (account_id, username, user_details['email']))
+
+        # account_ssh_keys
+        for key in user_details['ssh_keys']:
+
+          cur.execute("""select ssh_public_key from account_ssh_keys where
+            account_id = %s""", account_id)
+          db_keys = [r[0].strip() for r in cur.fetchall()]
+          if key.strip() not in db_keys:
+
+            cur.execute("""select max(seq)+1 from account_ssh_keys
+                                  where account_id = %s""", account_id)
+            seq = cur.fetchall()[0][0]
+            if seq is None:
+              seq = 1
+            cur.execute("""insert into account_ssh_keys
+                            (ssh_public_key, valid, account_id, seq)
+                            values
+                            (%s, 'Y', %s, %s)""",
+                            (key.strip(), account_id, seq))
+
+        # account_external_ids
+        ## external_id
+        if not cur.execute("""select account_id from account_external_ids
+                              where account_id = %s and external_id = %s""",
+                           (account_id, user_details['openid_external_id'])):
+          cur.execute("""insert into account_external_ids
+                         (account_id, email_address, external_id)
+                         values (%s, %s, %s)""",
+                     (account_id, user_details['email'], user_details['openid_external_id']))
+        if not cur.execute("""select account_id from account_external_ids
+                              where account_id = %s and external_id = %s""",
+                           (account_id, "username:%s" % username)):
+          cur.execute("""insert into account_external_ids
+                         (account_id, external_id) values (%s, %s)""",
+                      (account_id, "username:%s" % username))
+
+        if user_details.get('email', None) is not None:
+          if not cur.execute("""select account_id from account_external_ids
+                                where account_id = %s and external_id = %s""",
+                             (account_id, "mailto:%s" % user_details['email'])):
+            cur.execute("""insert into account_external_ids
+                           (account_id, email_address, external_id)
+                           values (%s, %s, %s)""",
+                        (account_id, user_details['email'], "mailto:%s" %
+                        user_details['email']))
+
+  if account_id is not None:
+    # account_group_members
+    # user_details['add_groups'] is a list of group names for which the
+    # user is either "Approved" or "Administrator"
+
+    groups_to_add = []
+    groups_to_watch = {}
+    groups_to_rm = {}
+
+    for group in user_details['add_groups']:
+      # if you are in the group nova-core, that should also put you in nova
+      add_groups = group_implies_groups[group_ids[group]]
+      add_groups.append(group_ids[group])
+      for add_group in add_groups:
+        if add_group not in groups_to_add:
+          groups_to_add.append(add_group)
+      # We only want to add watches for direct project membership groups
+      groups_to_watch[group_ids[group]] = group
+
+    # groups_to_add is now the full list of all groups we think the user
+    # should belong to. we want to limit the users groups to this list
+    for group in groups:
+      if group_ids[group] not in groups_to_add:
+        if group not in groups_to_rm.values():
+          groups_to_rm[group_ids[group]] = group
+
+    for group_id in groups_to_add:
+      if not cur.execute("""select account_id from account_group_members
+                            where account_id = %s and group_id = %s""",
+                         (account_id, group_id)):
+        # The current user does not exist in the group. Add it.
+        cur.execute("""insert into account_group_members
+                         (account_id, group_id)
+                       values (%s, %s)""", (account_id, group_id))
+        os_project_name = groups_to_watch.get(group_id, None)
+        if os_project_name is not None:
+          if os_project_name.endswith("-core"):
+              os_project_name = os_project_name[:-5]
+          os_project_name = "openstack/%s" % os_project_name
+          if os_project_name in projects:
+            if not cur.execute("""select account_id
+                                   from account_project_watches
+                                  where account_id = %s
+                                    and project_name = %s""",
+                                 (account_id, os_project_name)):
+                cur.execute("""insert into account_project_watches
+                               VALUES
+                               ("Y", "N", "N", %s, %s, "*")""",
+                               (account_id, os_project_name))
+
+    for (group_id, group_name) in groups_to_rm.items():
+      cur.execute("""delete from account_group_members
+                     where account_id = %s and group_id = %s""",
+                  (account_id, group_id))
+
+os.system("ssh -i %s -p29418 %s@localhost gerrit flush-caches" %
+          (GERRIT_SSH_KEY, GERRIT_USER))
+
+conn.commit()

modules/gerrit/files/scripts/update_users.py

@@ -0,0 +1,430 @@
+#! /usr/bin/env python
+# Copyright (C) 2012 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# Synchronize Gerrit users from Launchpad.
+# TODO items:
+# 1. add a temporary (instance level) object store for the launchpad class
+# 2. split out the two classes into separate files to be used as a library
+
+import os
+import ConfigParser
+import StringIO
+import paramiko
+import json
+import logging
+import uuid
+from launchpadlib.launchpad import Launchpad
+from launchpadlib.uris import LPNET_SERVICE_ROOT
+
+from datetime import datetime
+
+from openid.consumer import consumer
+from openid.cryptutil import randomString
+
+GERRIT_USER = os.environ.get('GERRIT_USER', 'launchpadsync')
+GERRIT_CONFIG = os.environ.get('GERRIT_CONFIG',
+                                 '/home/gerrit2/review_site/etc/gerrit.config')
+GERRIT_SECURE_CONFIG = os.environ.get('GERRIT_SECURE_CONFIG',
+                                 '/home/gerrit2/review_site/etc/secure.config')
+GERRIT_SSH_KEY = os.environ.get('GERRIT_SSH_KEY',
+                                 '/home/gerrit2/.ssh/launchpadsync_rsa')
+GERRIT_CACHE_DIR = os.path.expanduser(os.environ.get('GERRIT_CACHE_DIR',
+                                '~/.launchpadlib/cache'))
+GERRIT_CREDENTIALS = os.path.expanduser(os.environ.get('GERRIT_CREDENTIALS',
+                                '~/.launchpadlib/creds'))
+GERRIT_BACKUP_PATH = os.environ.get('GERRIT_BACKUP_PATH',
+                                '/home/gerrit2/dbupdates')
+
+logging.basicConfig(format='%(asctime)-6s: %(name)s - %(levelname)s - %(message)s', filename='/var/log/gerrit/update_users.log')
+logger= logging.getLogger('update_users')
+logger.setLevel(logging.INFO)
+
+for check_path in (os.path.dirname(GERRIT_CACHE_DIR),
+                   os.path.dirname(GERRIT_CREDENTIALS),
+                   GERRIT_BACKUP_PATH):
+  if not os.path.exists(check_path):
+    os.makedirs(check_path)
+
+def get_broken_config(filename):
+  """ gerrit config ini files are broken and have leading tabs """
+  text = ""
+  with open(filename,"r") as conf:
+    for line in conf.readlines():
+      text = "%s%s" % (text, line.lstrip())
+
+  fp = StringIO.StringIO(text)
+  c=ConfigParser.ConfigParser()
+  c.readfp(fp)
+  return c
+
+gerrit_config = get_broken_config(GERRIT_CONFIG)
+secure_config = get_broken_config(GERRIT_SECURE_CONFIG)
+
+DB_USER = gerrit_config.get("database", "username")
+DB_PASS = secure_config.get("database","password")
+DB_DB = gerrit_config.get("database","database")
+
+def make_db_backup():
+  db_backup_file = "%s.%s.sql" % (DB_DB, datetime.isoformat(datetime.now()))
+  db_backup_path = os.path.join(GERRIT_BACKUP_PATH, db_backup_file)
+  retval = os.system("mysqldump --opt -u%s -p%s %s > %s" %
+                       (DB_USER, DB_PASS, DB_DB, db_backup_path))
+  if retval != 0:
+    logger.error("Problem taking a db dump, aborting db update")
+    sys.exit(retval)
+
+class LaunchpadAction(object):
+  def __init__(self):
+    logger.info('Connecting to Launchpad')
+    self.launchpad= Launchpad.login_with('Gerrit User Sync', LPNET_SERVICE_ROOT,
+                                 GERRIT_CACHE_DIR,
+                                 credentials_file = GERRIT_CREDENTIALS)
+
+    logger.info('Getting Launchpad teams')
+    self.lp_teams= self.get_all_sub_teams('openstack', [])
+
+  def get_all_sub_teams(self, team, have_teams):
+    for sub_team in self.launchpad.people[team].sub_teams:
+      if sub_team.name not in have_teams:
+         have_teams = self.get_all_sub_teams(sub_team.name, have_teams)
+    have_teams.append(team)
+    return have_teams
+
+  def get_sub_teams(self, team):
+    sub_teams= []
+    for sub_team in self.launchpad.people[team].sub_teams:
+      sub_teams.append(sub_team.name)
+    return sub_teams
+
+  def get_teams(self):
+    return self.lp_teams
+
+  def get_all_users(self):
+    logger.info('Getting Launchpad users')
+    users= []
+    for team in self.lp_teams:
+      for detail in self.launchpad.people[team].members_details:
+        if (detail.status == 'Approved' or detail.status == 'Administrator'):
+          name= detail.self_link.split('/')[-1]
+          if ((users.count(name) == 0) and (name not in self.lp_teams)):
+            users.append(name)
+    return users
+
+  def get_user_data(self, user):
+    return self.launchpad.people[user]
+
+  def get_team_members(self, team, gerrit):
+    users= []
+    for detail in self.launchpad.people[team].members_details:
+      if (detail.status == 'Approved' or detail.status == 'Administrator'):
+        name= detail.self_link.split('/')[-1]
+        # if we found a subteam
+        if name in self.lp_teams:
+          # check subteam for implied subteams
+          for implied_group in gerrit.get_implied_groups(name):
+            if implied_group in self.lp_teams:
+              users.extend(self.get_team_members(implied_group, gerrit))
+          users.extend(self.get_team_members(name, gerrit))
+          continue
+        users.append(name)
+    # check team for implied teams
+    for implied_group in gerrit.get_implied_groups(team):
+      if implied_group in self.lp_teams:
+        users.extend(self.get_team_members(implied_group, gerrit))
+    # filter out dupes
+    users= list(set(users))
+    return users
+
+  def get_team_watches(self, team):
+    users= []
+    for detail in self.launchpad.people[team].members_details:
+      if (detail.status == 'Approved' or detail.status == 'Administrator'):
+        name= detail.self_link.split('/')[-1]
+        if name in self.lp_teams:
+          continue
+        if users.count(name) == 0:
+          users.append(name)
+    return users
+
+  def get_team_display_name(self, team):
+    team_data = self.launchpad.people[team]
+    return team_data.display_name
+
+class GerritAction(object):
+  def __init__(self):
+    logger.info('Connecting to Gerrit')
+    self.ssh= paramiko.SSHClient()
+    self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    self.ssh.connect('localhost', username=GERRIT_USER, port=29418, key_filename=GERRIT_SSH_KEY)
+
+  def cleanup(self):
+    logger.info('Closing connection to Gerrit')
+    self.ssh.close()
+
+  def run_query(self, query):
+    command= 'gerrit gsql --format JSON -c "{0}"'.format(query)
+    stdin, stdout, stderr= self.ssh.exec_command(command)
+#   trying to get stdout return code or stderr can hang with large result sets
+#    for line in stderr:
+#      logger.error(line)
+    return stdout
+
+  def get_groups(self):
+    logger.info('Getting Gerrit groups')
+    groups= []
+    query= "select name from account_groups"
+    stdout= self.run_query(query)
+    for line in stdout:
+      row= json.loads(line)
+      if row['type'] == 'row':
+        group= row['columns']['name']
+        groups.append(group)
+    return groups
+
+  def get_users(self):
+    logger.info('Getting Gerrit users')
+    users= []
+    query= "select external_id from account_external_ids"
+    stdout= self.run_query(query)
+    for line in stdout:
+      row= json.loads(line)
+      if row['type'] == 'row':
+        user= row['columns']['external_id'].replace('username:','')
+        users.append(user)
+    return users
+
+  def get_group_id(self, group_name):
+    query= "select group_id from account_groups where name='{0}'".format(group_name)
+    stdout= self.run_query(query)
+    line= stdout.readline()
+    row= json.loads(line)
+    if row['type'] == 'row':
+      return row['columns']['group_id']
+    else:
+      return 0
+
+  def get_user_id(self, user_name):
+    query= "select account_id from account_external_ids where external_id='username:{0}'".format(user_name)
+    stdout= self.run_query(query)
+    line= stdout.readline()
+    row= json.loads(line)
+    return row['columns']['account_id']
+
+  def get_users_from_group(self, group_name):
+    logger.info('Getting Gerrit users from group %s', group_name)
+    users= []
+    gid= self.get_group_id(group_name)
+
+    query= "select external_id from account_external_ids join account_group_members on account_group_members.account_id=account_external_ids.account_id where account_group_members.group_id={0} and external_id like 'username%%'".format(gid)
+    stdout= self.run_query(query)
+    for line in stdout:
+      row= json.loads(line)
+      if row['type'] == 'row':
+        user= row['columns']['external_id'].replace('username:','')
+        users.append(user)
+    return users
+
+  def get_users_from_watches(self, group_name):
+    logger.info('Getting Gerrit users from watch list %s', group_name)
+    users= []
+    if group_name.endswith("-core"):
+      group_name = group_name[:-5]
+    group_name = "openstack/{0}".format(group_name)
+
+    query= "select external_id from account_external_ids join account_project_watches on account_project_watches.account_id=account_external_ids.account_id where account_project_watches.project_name like '{0}' and external_id like 'username%%'".format(group_name)
+    stdout= self.run_query(query)
+    for line in stdout:
+      row= json.loads(line)
+      if row['type'] == 'row':
+        user= row['columns']['external_id'].replace('username:','')
+        users.append(user)
+    return users
+
+
+  def get_implied_groups(self, group_name):
+    gid= self.get_group_id(group_name)
+    groups= []
+    query= "select name from account_groups join account_group_includes on account_group_includes.include_id=account_groups.group_id where account_group_includes.group_id={0}".format(gid)
+    stdout= self.run_query(query)
+    for line in stdout:
+      row= json.loads(line)
+      if row['type'] == 'row':
+        group= row['columns']['name']
+        groups.append(group)
+    return groups
+
+  def add_group(self, group_name, group_display_name):
+    logger.info('New group %s (%s)', group_display_name, group)
+    query= "insert into account_group_id (s) values (NULL)"
+    stdout= self.run_query(query)
+    row= json.loads(stdout.readline())
+    if row['rowCount'] is not 1:
+      print "Could not get a new account group ID"
+      raise
+    query= "select max(s) from account_group_id"
+    stdout= self.run_query(query)
+    row= json.loads(stdout.readline())
+    gid= row['columns']['max(s)']
+    full_uuid= "{0}{1}".format(uuid.uuid4().hex, uuid.uuid4().hex[:8])
+    query= "insert into account_groups (group_id, group_type, owner_group_id, name, description, group_uuid) values ({0}, 'INTERNAL', 1, '{1}', '{2}', '{3}')". format(gid, group_name, group_display_name, full_uuid)
+    self.run_query(query)
+    query= "insert into account_group_names (group_id, name) values ({0}, '{1}')".format(gid, group_name)
+    self.run_query(query)
+
+  def add_user(self, user_name, user_data):
+    logger.info("Adding Gerrit user %s", user_name)
+    openid_consumer = consumer.Consumer(dict(id=randomString(16, '0123456789abcdef')), None)
+    openid_request = openid_consumer.begin("https://launchpad.net/~%s" % user_data.name)
+    user_openid_external_id = openid_request.endpoint.getLocalID()
+    query= "select account_id from account_external_ids where external_id in ('{0}')".format(user_openid_external_id)
+    stdout= self.run_query(query)
+    row= json.loads(stdout.readline())
+    if row['type'] == 'row':
+      # we have a result so this is an updated user name
+      account_id= row['columns']['account_id']
+      query= "update account_external_ids set external_id='{0}' where external_id like 'username%%' and account_id = {1}".format('username:%s' % user_name, account_id)
+      self.run_query(query)
+    else:
+      # we really do have a new user
+      user_ssh_keys= ["%s %s %s" % ('ssh-%s' % key.keytype.lower(), key.keytext, key.comment) for key in user_data.sshkeys]
+      user_email= None
+      try:
+        email = user_data.preferred_email_address.email
+      except ValueError:
+        pass
+      query= "insert into account_id (s) values (NULL)"
+      self.run_query(query)
+      query= "select max(s) from account_id"
+      stdout= self.run_query(query)
+      row= json.loads(stdout.readline())
+      uid= row['columns']['max(s)']
+      query= "insert into accounts (account_id, full_name, preferred_email) values ({0}, '{1}', '{2}')".format(uid, user_name, user_email)
+      self.run_query(query)
+      keyno= 1
+      for key in user_ssh_keys:
+        query= "insert into account_ssh_keys (ssh_public_key, valid, account_id, seq) values ('{0}', 'Y', {1}, {2})".format(key.strip(), uid, keyno)
+        self.run_query(query)
+        keyno = keyno + 1
+      query= "insert into account_external_ids (account_id, email_address, external_id) values ({0}, '{1}', '{2}')".format(uid, user_email, user_openid_external_id)
+      self.run_query(query)
+      query= "insert into account_external_ids (account_id, external_id) values ({0}, '{1}')".format(uid, "username:%s" % user_name)
+      self.run_query(query)
+      if user_email is not None:
+        query= "insert into account_external_ids (account_id, email_address, external_id) values ({0}. '{1}', '{2}')".format(uid, user_email, "mailto:%s" % user_email)
+    return None
+
+  def add_user_to_group(self, user_name, group_name):
+    logger.info("Adding Gerrit user %s to group %s", user_name, group_name)
+    uid= self.get_user_id(user_name)
+    gid= self.get_group_id(group_name)
+    if gid is 0:
+      print "Trying to add user {0} to non-existent group {1}".format(user_name, group_name)
+      raise
+    query= "insert into account_group_members (account_id, group_id) values ({0}, {1})".format(uid, gid)
+    self.run_query(query)
+
+  def add_user_to_watch(self, user_name, group_name):
+    logger.info("Adding Gerrit user %s to watch group %s", user_name, group_name)
+    uid= self.get_user_id(user_name)
+    if group_name.endswith("-core"):
+      group_name = group_name[:-5]
+    group_name = "openstack/{0}".format(group_name)
+    query= "insert into account_project_watches VALUES ('Y', 'N', 'N', {0}, '{1}', '*')". format(uid, group_name)
+    self.run_query(query)
+
+
+  def del_user_from_group(self, user_name, group_name):
+    logger.info("Deleting Gerrit user %s from group %s", user_name, group_name)
+    uid= self.get_user_id(user_name)
+    gid= self.get_group_id(group_name)
+    query= "delete from account_group_members where account_id = {0} and group_id = {1}".format(uid, gid)
+    self.run_query(query)
+    if group_name.endswith("-core"):
+      group_name = group_name[:-5]
+    group_name= "openstack/{0}".format(group_name)
+    query= "delete from account_project_watches where account_id = {0} and project_name= '{1}'".format(uid, group_name)
+    self.run_query(query)
+
+  def rebuild_sub_groups(self, group, sub_groups):
+    gid= self.get_group_id(group)
+    for sub_group in sub_groups:
+      sgid= self.get_group_id(sub_group)
+      query= "select group_id from account_group_includes where group_id={0} and include_id={1}".format(gid, sgid)
+      stdout= self.run_query(query)
+      row= json.loads(stdout.readline())
+      if row['type'] != 'row':
+        logger.info('Adding implied group %s to group %s', group, sub_group)
+        query= "insert into account_group_includes (group_id, include_id) values ({0}, {1})".format(gid, sgid)
+        self.run_query(query)
+
+
+# Actual work starts here!
+
+lp= LaunchpadAction()
+gerrit= GerritAction()
+
+logger.info('Making DB backup')
+make_db_backup()
+
+logger.info('Starting group reconcile')
+lp_groups= lp.get_teams()
+gerrit_groups= gerrit.get_groups()
+
+group_diff= filter(lambda a: a not in gerrit_groups, lp_groups)
+for group in group_diff:
+  group_display_name= lp.get_team_display_name(group)
+  gerrit.add_group(group, group_display_name)
+
+for group in lp_groups:
+  sub_group= lp.get_sub_teams(group)
+  if sub_group:
+    gerrit.rebuild_sub_groups(group, sub_group)
+
+logger.info('End group reconcile')
+
+logger.info('Starting user reconcile')
+lp_users= lp.get_all_users()
+gerrit_users= gerrit.get_users()
+
+user_diff= filter(lambda a: a not in gerrit_users, lp_users)
+for user in user_diff:
+  gerrit.add_user(user, lp.get_user_data(user))
+
+logger.info('End user reconcile')
+
+logger.info('Starting user to group reconcile')
+lp_groups= lp.get_teams()
+for group in lp_groups:
+  # First find users to attach to groups
+  gerrit_group_users= gerrit.get_users_from_group(group)
+  lp_group_users= lp.get_team_members(group, gerrit)
+
+  group_diff= filter(lambda a: a not in gerrit_group_users, lp_group_users)
+  for user in group_diff:
+    gerrit.add_user_to_group(user, group)
+  # Second find users to attach to watches
+  lp_group_watches= lp.get_team_watches(group)
+  gerrit_group_watches= gerrit.get_users_from_watches(group)
+  group_diff= filter(lambda a: a not in gerrit_group_watches, lp_group_watches)
+  for user in group_diff:
+    gerrit.add_user_to_watch(user, group)
+  # Third find users to remove from groups/watches
+  group_diff= filter(lambda a: a not in lp_group_users, gerrit_group_users)
+  for user in group_diff:
+    gerrit.del_user_from_group(user, group)
+
+logger.info('Ending user to group reconcile')
+
+gerrit.cleanup()

modules/gerrit/manifests/init.pp

@@ -109,29 +109,27 @@ class gerrit($virtual_hostname='',
 
   # Skip cron jobs if we're in test mode
   if ($testmode == false) {
-    cron { "gerritupdateci":
-      user => gerrit2,
-      minute => "*/15",
-      command => 'sleep $((RANDOM\%60)) && cd /home/gerrit2/openstack-ci && /usr/bin/git pull -q origin master'
-    }
 
     cron { "gerritsyncusers":
       user => gerrit2,
       minute => "*/15",
-      command => 'sleep $((RANDOM\%60+60)) && cd /home/gerrit2/openstack-ci && python gerrit/update_gerrit_users.py'
+      command => 'sleep $((RANDOM\%60+60)) && python /usr/local/gerrit/scripts/update_gerrit_users.py',
+      require => File['/usr/local/gerrit/scripts'],
     }
 
     cron { "gerritclosepull":
       user => gerrit2,
       minute => "*/5",
-      command => 'sleep $((RANDOM\%60+90)) && cd /home/gerrit2/openstack-ci && python gerrit/close_pull_requests.py'
+      command => 'sleep $((RANDOM\%60+90)) && python /usr/local/gerrit/scripts/close_pull_requests.py',
+      require => File['/usr/local/gerrit/scripts'],
     }
 
     cron { "expireoldreviews":
       user => gerrit2,
       hour => 6,
       minute => 3,
-      command => 'cd /home/gerrit2/openstack-ci && python gerrit/expire_old_reviews.py'
+      command => 'python /usr/local/gerrit/scripts/expire_old_reviews.py',
+      require => File['/usr/local/gerrit/scripts'],
     }
 
     cron { "gerrit_repack":
@@ -494,4 +492,23 @@ class gerrit($virtual_hostname='',
       require => File['/etc/init.d/gerrit'],
       refreshonly => true,
   }
+
+  file { '/usr/local/gerrit':
+    owner => 'root',
+    group => 'root',
+    mode => 755,
+    ensure => 'directory',
+  }
+
+  file { '/usr/local/gerrit/scripts':
+    owner => 'root',
+    group => 'root',
+    mode => 755,
+    ensure => 'directory',
+    recurse => true,
+    require => File['/usr/local/gerrit'],
+    source => [
+                "puppet:///modules/gerrit/scripts",
+              ],
+  }
 }