opendev/system-config
Code Issues Proposed changes

Rework user sync script.

Browse Source 
identify_unused_accounts.py is a one-time script to be run manually
to clean up the database to make the sync script run much faster.

Change-Id: I0e954f86dd6bb2fedd6ea65aa9be3d5188e04eab
Reviewed-on: https://review.openstack.org/17169
Reviewed-by: Monty Taylor <mordred@inaugust.com>
Reviewed-by: Clark Boylan <clark.boylan@gmail.com>
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Approved: James E. Blair <corvus@inaugust.com>
Tested-by: Jenkins
This commit is contained in:
James E. Blair 2012-11-14 08:06:16 -08:00 committed by Jenkins
parent 60050bea90
commit eae0e69432
4 changed files with 654 additions and 277 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

144
modules/launchpad_sync/files/identify_unused_accounts.py Normal file
View File

@ -0,0 +1,144 @@
#! /usr/bin/env python
# Copyright (C) 2011 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# Synchronize Gerrit users from Launchpad.
#SELECT account_id FROM accounts LEFT OUTER JOIN changes ON (changes.owner_account_id = accounts.account_id) WHERE changes.owner_account_id IS NULL;
#SELECT account_id FROM accounts LEFT OUTER JOIN change_messages ON (change_messages.author_id = accounts.account_id) WHERE change_messages.author_id IS NULL;
import ConfigParser
import MySQLdb
import StringIO
import os
def get_broken_config(filename):
""" gerrit config ini files are broken and have leading tabs """
text = ""
with open(filename, "r") as conf:
for line in conf.readlines():
text = "%s%s" % (text, line.lstrip())
fp = StringIO.StringIO(text)
c = ConfigParser.ConfigParser()
c.readfp(fp)
return c
GERRIT_CONFIG = os.environ.get('GERRIT_CONFIG',
'/home/gerrit2/review_site/etc/gerrit.config')
GERRIT_SECURE_CONFIG = os.environ.get(
'GERRIT_SECURE_CONFIG',
'/home/gerrit2/review_site/etc/secure.config')
gerrit_config = get_broken_config(GERRIT_CONFIG)
secure_config = get_broken_config(GERRIT_SECURE_CONFIG)
DB_USER = gerrit_config.get("database", "username")
DB_PASS = secure_config.get("database", "password")
DB_DB = gerrit_config.get("database", "database")
conn = MySQLdb.connect(user=DB_USER, passwd=DB_PASS, db=DB_DB)
cur = conn.cursor()
account_ids = []
cur.execute("select distinct account_id from account_external_ids where external_id like 'http%'")
for row in cur.fetchall():
account_ids.append(row[0])
print len(account_ids)
cur.execute("select distinct owner_account_id from changes")
for row in cur.fetchall():
if row[0] in account_ids:
account_ids.remove(row[0])
print len(account_ids)
cur.execute("select distinct author_id from change_messages")
for row in cur.fetchall():
if row[0] in account_ids:
account_ids.remove(row[0])
print len(account_ids)
cur.execute("select distinct account_id from patch_set_approvals")
for row in cur.fetchall():
if row[0] in account_ids:
account_ids.remove(row[0])
print len(account_ids)
cur.execute("select distinct author_id from patch_comments")
for row in cur.fetchall():
if row[0] in account_ids:
account_ids.remove(row[0])
print len(account_ids)
cur.execute("select distinct uploader_account_id from patch_sets")
for row in cur.fetchall():
if row[0] in account_ids:
account_ids.remove(row[0])
print len(account_ids)
cur.execute("select group_id from account_group_names "
"where name='openstack-cla'")
gid = cur.fetchall()[0][0]
cur.execute("select account_id from account_group_members where group_id=%s ",
(gid))
for row in cur.fetchall():
if row[0] in account_ids:
account_ids.remove(row[0])
print len(account_ids)
for account_id in account_ids:
cur.execute("delete from account_agreements where account_id=%s",
(account_id))
cur.execute("delete from account_diff_preferences where id=%s",
(account_id))
cur.execute("delete from account_external_ids where account_id=%s",
(account_id))
cur.execute("delete from account_group_members where account_id=%s",
(account_id))
cur.execute("delete from account_group_members_audit where account_id=%s",
(account_id))
cur.execute("delete from account_patch_reviews where account_id=%s",
(account_id))
cur.execute("delete from account_project_watches where account_id=%s",
(account_id))
cur.execute("delete from account_ssh_keys where account_id=%s",
(account_id))
cur.execute("delete from accounts where account_id=%s",
(account_id))
print len(account_ids)
groups_to_del = []
cur.execute("select * from account_group_names")
for row in cur.fetchall():
if row[1].endswith('-core'): continue
if row[1].endswith('-admins'): continue
if row[1].endswith('-drivers'): continue
if row[1] == 'openstack-cla': continue
if row[1] == 'openstack-release': continue
if row[1] == 'heat': continue
if row[1][0] >= 'A' and row[1][0] <= 'Z': continue
print 'delete group', row[1]
groups_to_del.append(row[0])
for gid in groups_to_del:
cur.execute("delete from account_group_includes where group_id=%s", (gid))
cur.execute("delete from account_group_includes where include_id=%s", (gid))
cur.execute("delete from account_group_members where group_id=%s", (gid))
cur.execute("delete from account_group_members_audit where group_id=%s",
(gid))
cur.execute("delete from account_group_names where group_id=%s", (gid))
cur.execute("delete from account_groups where group_id=%s", (gid))
conn.commit()

781
modules/launchpad_sync/files/update_gerrit_users.py
View File

@ -15,6 +15,7 @@
# Synchronize Gerrit users from Launchpad. # Synchronize Gerrit users from Launchpad.
import fcntl import fcntl
import logging import logging
import logging.config import logging.config
@ -22,6 +23,7 @@ import os
import subprocess import subprocess
import sys import sys
import uuid import uuid
import cPickle as pickle
from datetime import datetime from datetime import datetime
@ -41,8 +43,6 @@ from launchpadlib.uris import LPNET_SERVICE_ROOT
from openid.consumer import consumer from openid.consumer import consumer
from openid.cryptutil import randomString from openid.cryptutil import randomString
DEBUG = False
# suppress pyflakes # suppress pyflakes
pkg_resources.get_supported_platform() pkg_resources.get_supported_platform()
@ -57,12 +57,13 @@ except IOError:
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('user', help='The gerrit admin user') parser.add_argument('user', help='The gerrit admin user')
parser.add_argument('ssh_key', help='The gerrit admin SSH key file') parser.add_argument('ssh_key', help='The gerrit admin SSH key file')
parser.add_argument('site', parser.add_argument('log_config', default=None, nargs='?',
help='The site in use (typically openstack or stackforge)')
parser.add_argument('root_team', help='The root launchpad team to pull from')
parser.add_argument('log_config',
default=None,
help='Path to file containing logging config') help='Path to file containing logging config')
parser.add_argument('--prep-only', action='store_true')
parser.add_argument('--skip-prep', action='store_true')
parser.add_argument('--skip-dump', action='store_true')
parser.add_argument('-d', action='store_true')
options = parser.parse_args() options = parser.parse_args()
GERRIT_USER = options.user GERRIT_USER = options.user
@ -88,19 +89,22 @@ def setup_logging():
raise Exception("Unable to read logging config file at %s" % fp) raise Exception("Unable to read logging config file at %s" % fp)
logging.config.fileConfig(fp) logging.config.fileConfig(fp)
else: else:
logging.basicConfig(filename='/home/gerrit2/gerrit_user_sync.log', if options.d:
level=logging.DEBUG) logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
setup_logging() setup_logging()
log = logging.getLogger('gerrit_user_sync') log = logging.getLogger('gerrit_user_sync')
log.info('Gerrit user sync start ' + str(datetime.now())) log.info('Gerrit user sync start ' + str(datetime.now()))
for check_path in (os.path.dirname(GERRIT_CACHE_DIR), if not options.skip_dump:
os.path.dirname(GERRIT_CREDENTIALS), for check_path in (os.path.dirname(GERRIT_CACHE_DIR),
GERRIT_BACKUP_PATH): os.path.dirname(GERRIT_CREDENTIALS),
if not os.path.exists(check_path): GERRIT_BACKUP_PATH):
log.info('mkdir ' + check_path) if not os.path.exists(check_path):
os.makedirs(check_path) log.info('mkdir ' + check_path)
os.makedirs(check_path)
def get_broken_config(filename): def get_broken_config(filename):
@ -131,12 +135,13 @@ DB_DB = gerrit_config.get("database", "database")
db_backup_file = "%s.%s.sql" % (DB_DB, datetime.isoformat(datetime.now())) db_backup_file = "%s.%s.sql" % (DB_DB, datetime.isoformat(datetime.now()))
db_backup_path = os.path.join(GERRIT_BACKUP_PATH, db_backup_file) db_backup_path = os.path.join(GERRIT_BACKUP_PATH, db_backup_file)
log.info('Backup mysql DB to ' + db_backup_path) if not options.skip_dump:
retval = os.system("mysqldump --opt -u%s -p%s %s | gzip -9 > %s.gz" % log.info('Backup mysql DB to ' + db_backup_path)
(DB_USER, DB_PASS, DB_DB, db_backup_path)) retval = os.system("mysqldump --opt -u%s -p%s %s | gzip -9 > %s.gz" %
if retval != 0: (DB_USER, DB_PASS, DB_DB, db_backup_path))
print "Problem taking a db dump, aborting db update" if retval != 0:
sys.exit(retval) print "Problem taking a db dump, aborting db update"
sys.exit(retval)
log.info('Connect to mysql DB') log.info('Connect to mysql DB')
conn = MySQLdb.connect(user=DB_USER, passwd=DB_PASS, db=DB_DB) conn = MySQLdb.connect(user=DB_USER, passwd=DB_PASS, db=DB_DB)
@ -145,83 +150,341 @@ cur = conn.cursor()
log.info('Connecting to launchpad') log.info('Connecting to launchpad')
launchpad = Launchpad.login_with('Gerrit User Sync', LPNET_SERVICE_ROOT, launchpad = Launchpad.login_with('Gerrit User Sync', LPNET_SERVICE_ROOT,
GERRIT_CACHE_DIR, GERRIT_CACHE_DIR,
credentials_file=GERRIT_CREDENTIALS) credentials_file=GERRIT_CREDENTIALS,
version='devel')
log.info('Connected to launchpad') log.info('Connected to launchpad')
def get_sub_teams(team, have_teams): class Group(object):
for sub_team in launchpad.people[team].sub_teams: def __init__(self, name, id):
if sub_team.name not in have_teams: self.name = name
have_teams = get_sub_teams(sub_team.name, have_teams) self.id = id
have_teams.append(team)
return have_teams
class Team(object):
def __init__(self, name, display_name):
self.name = name
self.display_name = display_name
self.members = []
self.sub_teams = []
log.info('Getting teams') class LPUser(object):
teams_todo = get_sub_teams(options.root_team, []) def __init__(self, name):
self.name = name
self.display_name = None
self.email = None
self.ssh_keys = []
self.teams = []
self.openids = []
log.info('Listing projects') class GerritUser(object):
users = {} def __init__(self, id):
groups = {} self.id = id
groups_in_groups = {} self.name = None
group_implies_groups = {} self.emails = []
group_ids = {} self.openids = []
projects = subprocess.check_output(['/usr/bin/ssh', '-p', '29418',
'-i', GERRIT_SSH_KEY,
'-l', GERRIT_USER, 'localhost',
'gerrit', 'ls-projects']).split('\n')
log.info('Examining teams') class Sync(object):
for team_todo in teams_todo: def __init__(self):
team = launchpad.people[team_todo] self.log = logging.getLogger('sync')
groups[team.name] = team.display_name self.cursor = cur
self.teams = {}
self.lp_users = {}
self.openids = {}
self.gerrit_users = {}
self.groups = {}
# Attempt to get nested group memberships. ~nova-core, for instance, is a def __getstate__(self):
# member of ~nova, so membership in ~nova-core should imply membership in d = self.__dict__.copy()
# ~nova del d['log']
group_in_group = groups_in_groups.get(team.name, {}) del d['cursor']
for subgroup in team.sub_teams: return d
group_in_group[subgroup.name] = 1
# We should now have a dictionary of the form {'nova': {'nova-core': 1}}
groups_in_groups[team.name] = group_in_group
for detail in team.members_details: def __setstate__(self, state):
user = None self.__dict__ = state
self.log = logging.getLogger('sync')
self.cursor = cur
# detail.self_link == def getProjectList(self):
# 'https://api.launchpad.net/1.0/~team/+member/${username}' self.log.info('Listing projects')
login = detail.self_link.split('/')[-1] if options.skip_dump:
projects = [
if users.has_key(login): 'heat-api/heat',
user = users[login] 'heat-api/python-heatclient',
'openstack-ci/devstack-gate',
'openstack-ci/gerrit',
'openstack-ci/gerrit-verification-status-plugin',
'openstack-ci/gerritbot',
'openstack-ci/gerritlib',
'openstack-ci/git-review',
'openstack-ci/jeepyb',
'openstack-ci/jenkins-job-builder',
'openstack-ci/lodgeit',
'openstack-ci/meetbot',
'openstack-ci/nose-html-output',
'openstack-ci/puppet-apparmor',
'openstack-ci/puppet-dashboard',
'openstack-ci/puppet-vcsrepo',
'openstack-ci/pypi-mirror',
'openstack-ci/zuul',
'openstack-dev/devstack',
'openstack-dev/openstack-nose',
'openstack-dev/openstack-qa',
'openstack-dev/pbr',
'openstack-dev/sandbox',
'openstack/api-site',
'openstack/ceilometer',
'openstack/cinder',
'openstack/compute-api',
'openstack/glance',
'openstack/horizon',
'openstack/identity-api',
'openstack/image-api',
'openstack/keystone',
'openstack/melange',
'openstack/netconn-api',
'openstack/nova',
'openstack/object-api',
'openstack/openstack-chef',
'openstack/openstack-ci',
'openstack/openstack-ci-puppet',
'openstack/openstack-manuals',
'openstack/openstack-planet',
'openstack/openstack-puppet',
'openstack/oslo-incubator',
'openstack/python-cinderclient',
'openstack/python-glanceclient',
'openstack/python-keystoneclient',
'openstack/python-melangeclient',
'openstack/python-novaclient',
'openstack/python-openstackclient',
'openstack/python-quantumclient',
'openstack/python-swiftclient',
'openstack/quantum',
'openstack/requirements',
'openstack/swift',
'openstack/tempest',
'openstack/volume-api',
'stackforge/MRaaS',
'stackforge/bufunfa',
'stackforge/diskimage-builder',
'stackforge/libra',
'stackforge/marconi',
'stackforge/moniker',
'stackforge/python-monikerclient',
'stackforge/python-reddwarfclient',
'stackforge/reddwarf',
'stackforge/reddwarf-integration',
]
else: else:
user = dict(add_groups=[]) projects = subprocess.check_output(['/usr/bin/ssh', '-p', '29418',
'-i', GERRIT_SSH_KEY,
'-l', GERRIT_USER, 'localhost',
'gerrit', 'ls-projects'])
projects = projects.split('\n')
self.projects = projects
status = detail.status def getGroups(self):
if (status == "Approved" or status == "Administrator"): self.log.info('Listing groups')
user['add_groups'].append(team.name) self.groups = {}
users[login] = user self.cursor.execute("select group_id, name from account_groups")
for row in self.cursor.fetchall():
id, name = row
self.groups[name] = Group(name, id)
# If we picked up subgroups that were not in our original list of groups def getOpenID(self, openid):
# make sure they get added person = launchpad.people.getByOpenIDIdentifier(identifier=openid)
for (supergroup, subgroups) in groups_in_groups.items(): if not person:
for group in subgroups.keys(): return
if group not in groups.keys(): lp_user = self.lp_users.get(person)
groups[group] = None if not lp_user:
lp_user = LPUser(person.name)
self.lp_users[person.name] = lp_user
if openid not in lp_user.openids:
lp_user.openids.append(openid)
self.openids[openid] = lp_user
log.info('Examining groups') def getGerritUsers(self):
# account_groups # Get a list of gerrit users and external ids
# groups is a dict of team name to team display name log.info('Getting gerrit users')
# here, for every group we have in that dict, we're building another dict of cur.execute("""select account_id, external_id
# group_name to group_id - and if the database doesn't already have the from account_external_ids""")
# group, we're adding it rows = cur.fetchall()
for (group_name, group_display_name) in groups.items(): for i, row in enumerate(rows):
if cur.execute("select group_id from account_groups where name = %s", account_id = row[0]
group_name): if account_id in self.gerrit_users:
group_ids[group_name] = cur.fetchall()[0][0] g_user = self.gerrit_users[account_id]
else: else:
cur.execute("""insert into account_group_id (s) values (NULL)""") g_user = GerritUser(account_id)
cur.execute("select max(s) from account_group_id") self.gerrit_users[account_id] = g_user
if row[1].startswith('mailto:'):
g_user.emails.append(row[1][len('mailto:'):])
elif row[1].startswith('username:'):
g_user.name = row[1][len('username:'):]
else:
g_user.openids.append(row[1])
self.getOpenID(row[1])
def prep(self):
self.getProjectList()
self.getGroups()
self.getTeams()
self.getLPUsers()
self.getGerritUsers()
def fixOpenIDs(self):
for g_user in self.gerrit_users.values()[:]:
account_names = {}
for openid in g_user.openids:
lp_user = self.openids.get(openid)
if not lp_user:
continue
account_names[lp_user.name] = openid
if len(account_names.keys()) == 0:
continue
elif len(account_names.keys()) == 1:
if account_names.keys()[0] != g_user.name:
self.renameAccount(g_user, account_names.keys()[0])
else:
for openid in g_user.openids:
lp_user = self.openids[openid]
if lp_user.name != g_user.name:
other_id = self.getGerritAccountID(lp_user.name)
other_g_user = self.gerrit_users.get(other_id)
if other_g_user:
self.moveOpenID(g_user, other_g_user, openid)
else:
self.removeOpenID(g_user, openid)
def getGerritAccountID(self, name):
if self.cursor.execute("""select account_id from account_external_ids
where external_id=%s""",
("username:%s" % name)):
return self.cursor.fetchall()[0][0]
return None
def renameAccount(self, g_user, name):
log.info('Rename %s %s to %s' % (g_user.name, g_user.id, name))
# if other account exists, move openids and delete username
# else, change username
other_id = self.getGerritAccountID(name)
if not other_id:
# update external ids username:
if g_user.name:
log.debug('Update external_id %s: %s -> %s' % (
g_user.id, name, g_user.name))
self.cursor.execute("""update account_external_ids
set external_id=%s where account_id=%s and external_id=%s""",
("username:%s" % name,
g_user.id,
"username:%s" % g_user.name))
else:
log.debug('Insert external_id %s: %s' % (
g_user.id, g_user.name))
self.cursor.execute("""insert into account_external_ids
(account_id, external_id)
values (%s, %s)""",
(g_user.id, "username:%s" % name))
g_user.name = name
else:
log.debug('Rename %s by moving openid' % g_user.id)
other_g_user = self.gerrit_users.get(other_id)
for openid in g_user.openids:
self.moveOpenID(g_user, other_g_user, openid)
def removeOpenID(self, g_user, openid):
log.info('Remove openid %s from %s' % (openid, g_user.name))
self.cursor.execute("""delete from account_external_ids
where account_id=%s and external_id=%s""",
(g_user.id, openid))
def moveOpenID(self, src_user, dest_user, openid):
log.info('Move openid %s from %s to %s ' % (openid, src_user.name,
dest_user.name))
self.cursor.execute("""select email_address from account_external_ids
where account_id=%s and external_id=%s""",
(src_user.id, openid))
email = self.cursor.fetchall()[0][0]
self.removeOpenID(src_user, openid)
self.cursor.execute("""insert into account_external_ids
(account_id, email_address, external_id)
values (%s, %s, %s)""",
(dest_user.id, email, openid))
def sync(self):
self.fixOpenIDs()
self.addSubGroups()
self.syncUsers()
def getTeams(self):
log.info('Getting teams')
for group in self.groups.values():
self.getTeam(group.name)
def getTeam(self, name):
if name in self.teams:
return
log.debug('Getting team %s' % name)
try:
lpteam = launchpad.people[name]
except:
return
team = Team(lpteam.name, lpteam.display_name)
self.teams[team.name] = team
sub_team_names = [sub_team.name for sub_team in lpteam.sub_teams]
for detail in lpteam.members_details:
if detail.status not in ["Approved", "Administrator"]:
continue
# detail.self_link ==
# 'https://api.launchpad.net/1.0/~team/+member/${username}'
login = detail.self_link.split('/')[-1]
if login in sub_team_names:
continue
user = self.lp_users.get(login)
if not user:
user = LPUser(login)
self.lp_users[login] = user
user.teams.append(team)
team.members.append(user)
for sub_team in lpteam.sub_teams:
self.getTeam(sub_team.name)
team.sub_teams.append(self.teams[sub_team.name])
def addGroupToGroup(self, child_group, parent_group):
try:
log.info('Adding group %s to %s' % (child_group.name,
parent_group.name))
cur.execute("""insert into account_group_includes
(group_id, include_id)
values (%s, %s)""",
(parent_group.id, child_group.id))
except MySQLdb.IntegrityError:
pass
def addSubGroups(self):
log.info('Add subgroups')
for team in self.teams.values():
group = self.groups.get(team.name)
if not group:
continue
for sub_team in team.sub_teams:
sub_group = self.groups.get(sub_team.name)
if not sub_group:
sub_group = self.createGroup(sub_team)
self.addGroupToGroup(sub_group, group)
def createGroup(self, team):
log.info('Create group %s' % team.name)
self.cursor.execute(
"""insert into account_group_id (s) values (NULL)""")
self.cursor.execute("select max(s) from account_group_id")
group_id = cur.fetchall()[0][0] group_id = cur.fetchall()[0][0]
# Match the 40-char 'uuid' that java is producing # Match the 40-char 'uuid' that java is producing
@ -229,169 +492,115 @@ for (group_name, group_display_name) in groups.items():
second_uuid = uuid.uuid4() second_uuid = uuid.uuid4()
full_uuid = "%s%s" % (group_uuid.hex, second_uuid.hex[:8]) full_uuid = "%s%s" % (group_uuid.hex, second_uuid.hex[:8])
log.info('Adding group %s' % group_name) log.debug('Adding group %s' % team.name)
cur.execute("""insert into account_groups self.cursor.execute("""insert into account_groups
(group_id, group_type, owner_group_id, (group_id, group_type, owner_group_id,
name, description, group_uuid) name, description, group_uuid)
values values
(%s, 'INTERNAL', 1, %s, %s, %s)""", (%s, 'INTERNAL', 1, %s, %s, %s)""",
(group_id, group_name, group_display_name, full_uuid)) (group_id, team.name, team.display_name,
cur.execute("""insert into account_group_names (group_id, name) values full_uuid))
(%s, %s)""", self.cursor.execute("""insert into account_group_names
(group_id, group_name)) (group_id, name) values (%s, %s)""",
(group_id, team.name))
group = Group(team.name, group_id)
self.groups[team.name] = group
return group
group_ids[group_name] = group_id def getGerritAccountId(self, username):
if cur.execute("""select account_id from account_external_ids where
external_id in (%s)""",
"username:%s" % username):
return cur.fetchall()[0][0]
return None
def getLPUsers(self):
log.info('Getting LP users')
# Get info about all of the LP team members who are not already
# in the db
for lp_user in self.lp_users.values():
account_id = self.getGerritAccountId(lp_user.name)
if not account_id:
self.getLPUser(lp_user)
def getLPUser(self, lp_user):
log.debug('Getting info about %s' % lp_user.name)
# only call this if we have no info for a user
member = launchpad.people[lp_user.name]
openid_consumer = consumer.Consumer(
dict(id=randomString(16, '0123456789abcdef')),
None)
openid_request = openid_consumer.begin(
"https://launchpad.net/~%s" % member.name)
openid = openid_request.endpoint.getLocalID()
lp_user.openids.append(openid)
self.openids[openid] = lp_user
# account_group_includes
# groups_in_groups should be a dict of dicts, where the key is the larger
# group and the inner dict is a list of groups that are members of the
# larger group. So {'nova': {'nova-core': 1}}
for (group_name, subgroups) in groups_in_groups.items():
for subgroup_name in subgroups.keys():
try: try:
log.info('Adding included group %s' % group_name) lp_user.email = member.preferred_email_address.email
cur.execute("""insert into account_group_includes except ValueError:
(group_id, include_id)
values (%s, %s)""",
(group_ids[group_name], group_ids[subgroup_name]))
except MySQLdb.IntegrityError:
pass pass
# Make a list of implied group membership for key in member.sshkeys:
# building a list which is the opposite of groups_in_group. Here lp_user.ssh_keys.append("%s %s %s" %
# group_implies_groups is a dict keyed by group_id containing a list of (get_type(key.keytype),
# group_ids of implied membership. SO: if nova is 1 and nova-core is 2: key.keytext, key.comment))
# {'2': [1]}
for group_id in group_ids.values():
total_groups = []
groups_todo = [group_id]
while len(groups_todo) > 0:
current_group = groups_todo.pop()
total_groups.append(current_group)
cur.execute("""select group_id from account_group_includes
where include_id = %s""", (current_group))
for row in cur.fetchall():
if row[0] != 1 and row[0] not in total_groups:
groups_todo.append(row[0])
group_implies_groups[group_id] = total_groups
if DEBUG: def createGerritUser(self, lp_user, skip_openids=False):
def get_group_name(in_group_id): log.info('Add %s to Gerrit DB.' % lp_user.name)
for (group_name, group_id) in group_ids.items(): cur.execute("""insert into account_id (s) values (NULL)""")
if group_id == in_group_id: cur.execute("select max(s) from account_id")
return group_name
print "groups in groups"
for (k, v) in groups_in_groups.items():
print k, v
print "group_imples_groups"
for (k, v) in group_implies_groups.items():
print get_group_name(k)
new_groups = []
for val in v:
new_groups.append(get_group_name(val))
print "\t", new_groups
for (username, user_details) in users.items():
log.info('Syncing user: %s' % username)
member = launchpad.people[username]
# accounts
account_id = None
if cur.execute("""select account_id from account_external_ids where
external_id in (%s)""", ("username:%s" % username)):
account_id = cur.fetchall()[0][0] account_id = cur.fetchall()[0][0]
# We have this bad boy
# all we need to do is update his group membership
else:
# We need details
if not member.is_team:
openid_consumer = consumer.Consumer( cur.execute("""insert into accounts
dict(id=randomString(16, '0123456789abcdef')), (account_id, full_name, preferred_email)
None) values (%s, %s, %s)""",
openid_request = openid_consumer.begin( (account_id, lp_user.name, lp_user.email))
"https://launchpad.net/~%s" % member.name)
user_details['openid_external_id'] = \
openid_request.endpoint.getLocalID()
# Handle username change g_user = GerritUser(account_id)
if cur.execute("""select account_id from account_external_ids where g_user.name = lp_user.name
external_id in (%s)""", g_user.emails.append(lp_user.email)
user_details['openid_external_id']): self.gerrit_users[account_id] = g_user
account_id = cur.fetchall()[0][0]
log.info('Handling username change id %s to %s' %
(account_id, username))
cur.execute("""update account_external_ids
set external_id = %s
where external_id like 'username%%'
and account_id = %s""",
('username:%s' % username, account_id))
else:
email = None
try:
email = member.preferred_email_address.email
except ValueError:
pass
user_details['email'] = email
log.info('Add %s to Gerrit DB.' % username) # account_external_ids
cur.execute("""insert into account_id (s) values (NULL)""") ## external_id
cur.execute("select max(s) from account_id") if not skip_openids:
account_id = cur.fetchall()[0][0] for openid in lp_user.openids:
if not cur.execute("""select account_id
from account_external_ids
where account_id = %s
and external_id = %s""",
(account_id, openid)):
cur.execute("""insert into account_external_ids
(account_id, email_address, external_id)
values (%s, %s, %s)""",
(account_id, lp_user.email, openid))
cur.execute("""insert into accounts if not cur.execute("""select account_id
(account_id, full_name, preferred_email) from account_external_ids
where account_id = %s
and external_id = %s""",
(account_id, "username:%s" % lp_user.name)):
cur.execute("""insert into account_external_ids
(account_id, external_id)
values (%s, %s)""",
(account_id, "username:%s" % lp_user.name))
if lp_user.email:
if not cur.execute("""select account_id
from account_external_ids
where account_id = %s
and external_id = %s""",
(account_id, "mailto:%s" % lp_user.email)):
cur.execute("""insert into account_external_ids
(account_id, email_address, external_id)
values (%s, %s, %s)""", values (%s, %s, %s)""",
(account_id, username, user_details['email'])) (account_id, lp_user.email,
"mailto:%s" % lp_user.email))
# account_external_ids for key in lp_user.ssh_keys:
## external_id
if not cur.execute("""select account_id
from account_external_ids
where account_id = %s
and external_id = %s""",
(account_id,
user_details['openid_external_id'])):
cur.execute("""insert into account_external_ids
(account_id, email_address, external_id)
values (%s, %s, %s)""",
(account_id, user_details['email'],
user_details['openid_external_id']))
if not cur.execute("""select account_id
from account_external_ids
where account_id = %s
and external_id = %s""",
(account_id, "username:%s" % username)):
cur.execute("""insert into account_external_ids
(account_id, external_id)
values (%s, %s)""",
(account_id, "username:%s" % username))
if user_details.get('email', None) is not None:
if not cur.execute("""select account_id
from account_external_ids
where account_id = %s
and external_id = %s""",
(account_id, "mailto:%s" %
user_details['email'])):
cur.execute("""insert into account_external_ids
(account_id, email_address, external_id)
values (%s, %s, %s)""",
(account_id,
user_details['email'],
"mailto:%s" %
user_details['email']))
if account_id is not None:
# account_ssh_keys
log.info('Add ssh keys for %s' % username)
user_details['ssh_keys'] = ["%s %s %s" %
(get_type(key.keytype),
key.keytext,
key.comment)
for key in member.sshkeys]
for key in user_details['ssh_keys']:
cur.execute("""select ssh_public_key from account_ssh_keys where cur.execute("""select ssh_public_key from account_ssh_keys where
account_id = %s""", account_id) account_id = %s""", account_id)
db_keys = [r[0].strip() for r in cur.fetchall()] db_keys = [r[0].strip() for r in cur.fetchall()]
@ -406,67 +615,95 @@ for (username, user_details) in users.items():
values values
(%s, 'Y', %s, %s)""", (%s, 'Y', %s, %s)""",
(key.strip(), account_id, seq)) (key.strip(), account_id, seq))
return g_user
def addWatch(self, gerrit_user, group):
watch_name = group.name
if group.name.endswith("-core"):
watch_name = group.name[:-5]
if group.name.endswith("-drivers"):
watch_name = group.name[:-5]
for p in self.projects:
if watch_name in p:
watch_name = p
break
print watch_name
if watch_name in self.projects:
if not cur.execute("""select account_id
from account_project_watches
where account_id = %s
and project_name = %s""",
(gerrit_user.id, watch_name)):
cur.execute("""insert into account_project_watches
VALUES
("Y", "N", "N", %s, %s, "*")""",
(gerrit_user.id, watch_name))
def syncUsers(self):
for lp_user in self.lp_users.values():
g_id = self.getGerritAccountID(lp_user.name)
g_user = self.gerrit_users.get(g_id)
if not g_user:
g_user = self.createGerritUser(lp_user)
self.syncUser(lp_user, g_user)
def syncUser(self, lp_user, g_user):
log.debug('Syncing user: %s' % lp_user.name)
# account_group_members # account_group_members
# user_details['add_groups'] is a list of group names for which the # user_details['add_groups'] is a list of group names for which the
# user is either "Approved" or "Administrator" # user is either "Approved" or "Administrator"
groups_to_add = [] groups_to_add = []
groups_to_watch = {} groups_to_rm = []
groups_to_rm = {}
for group in user_details['add_groups']: for team in lp_user.teams:
# if you are in the group nova-core, that should also put you groups_to_add.append(self.groups[team.name])
# in nova
add_groups = group_implies_groups[group_ids[group]]
add_groups.append(group_ids[group])
for add_group in add_groups:
if add_group not in groups_to_add:
groups_to_add.append(add_group)
# We only want to add watches for direct project membership groups
groups_to_watch[group_ids[group]] = group
# groups_to_add is now the full list of all groups we think the user # groups_to_add is now the full list of all groups we think the user
# should belong to. we want to limit the users groups to this list # should belong to. we want to limit the users groups to this list
for group in groups: for group in self.groups.values():
if group_ids[group] not in groups_to_add: if group not in groups_to_add:
if group not in groups_to_rm.values(): if group not in groups_to_rm:
groups_to_rm[group_ids[group]] = group groups_to_rm.append(group)
for group_id in groups_to_add: for group in groups_to_add:
log.info('Add %s to group %s' % (username, group_id)) log.info('Add %s to group %s' % (lp_user.name, group.name))
if not cur.execute("""select account_id from account_group_members if not cur.execute("""select account_id from account_group_members
where account_id = %s and group_id = %s""", where account_id = %s and group_id = %s""",
(account_id, group_id)): (g_user.id, group.id)):
# The current user does not exist in the group. Add it. # The current user does not exist in the group. Add it.
cur.execute("""insert into account_group_members cur.execute("""insert into account_group_members
(account_id, group_id) (account_id, group_id)
values (%s, %s)""", (account_id, group_id)) values (%s, %s)""", (g_user.id, group.id))
os_project_name = groups_to_watch.get(group_id, None) self.addWatch(g_user, group)
if os_project_name is not None:
if os_project_name.endswith("-core"):
os_project_name = os_project_name[:-5]
os_project_name = \
"{site}/{project}".format(site=options.site,
project=os_project_name)
if os_project_name in projects:
if not cur.execute("""select account_id
from account_project_watches
where account_id = %s
and project_name = %s""",
(account_id, os_project_name)):
cur.execute("""insert into account_project_watches
VALUES
("Y", "N", "N", %s, %s, "*")""",
(account_id, os_project_name))
for (group_id, group_name) in groups_to_rm.items(): for group in groups_to_rm:
cur.execute("""delete from account_group_members cur.execute("""delete from account_group_members
where account_id = %s and group_id = %s""", where account_id = %s and group_id = %s""",
(account_id, group_id)) (g_user.id, group.id))
os.system("ssh -i %s -p29418 %s@localhost gerrit flush-caches" %
(GERRIT_SSH_KEY, GERRIT_USER)) if options.skip_prep and os.path.exists('/tmp/lpcache.pickle'):
log.info('Loading pickle')
out = open('/tmp/lpcache.pickle', 'rb')
sync = pickle.load(out)
out.close()
else:
log.info('Initializing')
sync = Sync()
sync.prep()
log.info('Saving pickle')
out = open('/tmp/lpcache.pickle', 'wb')
pickle.dump(sync, out, -1)
out.close()
if not options.prep_only:
log.info('Syncing')
sync.sync()
if not options.skip_dump:
os.system("ssh -i %s -p29418 %s@localhost gerrit flush-caches" %
(GERRIT_SSH_KEY, GERRIT_USER))
conn.commit() conn.commit()

4
modules/launchpad_sync/manifests/init.pp
View File

@ -1,9 +1,7 @@
# == Class: launchpad_sync # == Class: launchpad_sync
# #
class launchpad_sync( class launchpad_sync(
$root_team = '',
$script_logging_conf = '', $script_logging_conf = '',
$site = '',
$script_key_file = '/home/gerrit2/.ssh/id_rsa', $script_key_file = '/home/gerrit2/.ssh/id_rsa',
$script_user = 'update', $script_user = 'update',
$user = 'gerrit2' $user = 'gerrit2'
@ -31,7 +29,7 @@ class launchpad_sync(
cron { 'sync_launchpad_users': cron { 'sync_launchpad_users':
user => $user, user => $user,
minute => '*/15', minute => '*/15',
command => "sleep $((RANDOM\\%60+60)) && timeout -k 5m 8h python /usr/local/bin/update_gerrit_users.py ${script_user} ${script_key_file} ${site} ${root_team} ${script_logging_conf}", command => "sleep $((RANDOM\\%60+60)) && timeout -k 5m 8h python /usr/local/bin/update_gerrit_users.py ${script_user} ${script_key_file} ${script_logging_conf}",
require => File['/usr/local/bin/update_gerrit_users.py'], require => File['/usr/local/bin/update_gerrit_users.py'],
} }
} }

2
modules/openstack_project/manifests/gerrit.pp
View File

@ -190,8 +190,6 @@ class openstack_project::gerrit (
script_user => $script_user, script_user => $script_user,
script_key_file => $script_key_file, script_key_file => $script_key_file,
script_logging_conf => $script_logging_conf, script_logging_conf => $script_logging_conf,
site => 'openstack',
root_team => 'openstack',
} }
file { '/home/gerrit2/review_site/hooks/change-merged': file { '/home/gerrit2/review_site/hooks/change-merged':