From 383c023b1547b5f810b23024b0079d09b4f05269 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sun, 29 Jul 2012 12:54:27 -0500 Subject: [PATCH] Add support for initial project creation. If replicate_local is set, this will ensure that /var/lib/git is created, and that projects listed in the projects.config have repos there. Additionally, it creates a new config file, projects.config which is a yaml file listing all of the projects and various operational semantics about them, such as whether or not they should have pull requests closed and whether or not they track any remotes. This replaces remotes.config and github.config. Moving forward, there is no reason to not have this script be able to do github api calls to create the github repo if it's not there, set the github project description, gerrit api calls to create the project in gerrit, and initial project permissions templates. Change-Id: I1ad803b0aa5f7386206d0c3f4cd858017242fe64 --- modules/gerrit/files/scripts/fetch_remotes.py | 36 ++--- .../gerrit/files/scripts/make_local_repos.py | 67 ++++++++ modules/gerrit/manifests/init.pp | 23 +++ modules/gerrit/manifests/remotes.pp | 11 +- modules/gerrit/templates/remotes.config.erb | 7 - .../files/scripts/close_pull_requests.py | 33 ++-- modules/github/manifests/init.pp | 8 +- modules/github/templates/github.config.erb | 7 - .../files/review-dev.projects.yaml | 3 + .../files/review.projects.yaml | 145 ++++++++++++++++++ modules/openstack_project/manifests/gerrit.pp | 5 +- modules/openstack_project/manifests/init.pp | 117 -------------- modules/openstack_project/manifests/review.pp | 9 +- .../openstack_project/manifests/review_dev.pp | 5 +- 14 files changed, 276 insertions(+), 200 deletions(-) create mode 100755 modules/gerrit/files/scripts/make_local_repos.py delete mode 100644 modules/gerrit/templates/remotes.config.erb delete mode 100644 modules/github/templates/github.config.erb create mode 100644 modules/openstack_project/files/review-dev.projects.yaml create mode 100644 modules/openstack_project/files/review.projects.yaml diff --git a/modules/gerrit/files/scripts/fetch_remotes.py b/modules/gerrit/files/scripts/fetch_remotes.py index 776188e765..4dda133f3d 100755 --- a/modules/gerrit/files/scripts/fetch_remotes.py +++ b/modules/gerrit/files/scripts/fetch_remotes.py @@ -13,18 +13,19 @@ # License for the specific language governing permissions and limitations # under the License. -# Ensure that the specified remote exists in the repo and pull revisions -# from upstream +# Fetch remotes reads a project config file called projects.yaml +# It should look like: + +# - project: PROJECT_NAME +# options: +# - remote: https://gerrit.googlesource.com/gerrit -# [project "UPSTREAM_PROJECT"] -# remote = https://gerrit.googlesource.com/gerrit -import ConfigParser import logging import os -import re import subprocess import shlex +import yaml def run_command(cmd, status=False, env={}): cmd_list = shlex.split(str(cmd)) @@ -46,29 +47,22 @@ logging.basicConfig(level=logging.ERROR) REPO_ROOT = os.environ.get('REPO_ROOT', '/home/gerrit2/review_site/git') -REMOTES_CONFIG = os.environ.get('REMOTES_CONFIG', - '/home/gerrit2/remotes.config') +PROJECTS_YAML = os.environ.get('PROJECTS_YAML', + '/home/gerrit2/projects.yaml') -PROJECT_RE = re.compile(r'^project\s+"(.*)"$') +config = yaml.load(open(PROJECTS_YAML)) -config = ConfigParser.ConfigParser() -config.read(REMOTES_CONFIG) +for section in config: + project = section['project'] - -for section in config.sections(): - # Each section looks like [project "openstack/project"] - m = PROJECT_RE.match(section) - if not m: + if 'remote' not in section: continue - project = m.group(1) + project_git = "%s.git" % project os.chdir(os.path.join(REPO_ROOT, project_git)) - if not (config.has_option(section, "remote")): - continue - # Make sure that the specified remote exists - remote_url = config.get(section, "remote") + remote_url = section['remote'] # We could check if it exists first, but we're ignoring output anyway # So just try to make it, and it'll either make a new one or do nothing run_command("git remote add -f upstream %s" % remote_url) diff --git a/modules/gerrit/files/scripts/make_local_repos.py b/modules/gerrit/files/scripts/make_local_repos.py new file mode 100755 index 0000000000..f598446168 --- /dev/null +++ b/modules/gerrit/files/scripts/make_local_repos.py @@ -0,0 +1,67 @@ +#! /usr/bin/env python +# Copyright (C) 2011 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Make local repos reads a project config file called projects.yaml +# It should look like: + +# - project: PROJECT_NAME +# options: +# - close-pull +# remote: https://gerrit.googlesource.com/gerrit + +# TODO: add support for +# ssh -p 29418 localhost gerrit -name create-project PROJECT + +import logging +import os +import subprocess +import sys +import shlex +import yaml + +def run_command(cmd, status=False, env={}): + cmd_list = shlex.split(str(cmd)) + newenv = os.environ + newenv.update(env) + p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, env=newenv) + (out, nothing) = p.communicate() + if status: + return (p.returncode, out.strip()) + return out.strip() + + +def run_command_status(cmd, env={}): + return run_command(cmd, True, env) + + +logging.basicConfig(level=logging.ERROR) + +REPO_ROOT = sys.argv[1] +PROJECTS_YAML = os.environ.get('PROJECTS_YAML', + '/home/gerrit2/projects.yaml') + +config = yaml.load(open(PROJECTS_YAML)) + +for section in config: + project = section['project'] + + project_git = "%s.git" % project + project_dir = os.path.join(REPO_ROOT, project_git) + + if os.path.exists(project_dir): + continue + + run_command("git --bare init --shared=group %s" % project_dir) diff --git a/modules/gerrit/manifests/init.pp b/modules/gerrit/manifests/init.pp index 122a5f8401..d93b42c39d 100644 --- a/modules/gerrit/manifests/init.pp +++ b/modules/gerrit/manifests/init.pp @@ -70,6 +70,7 @@ class gerrit($virtual_hostname=$fqdn, $httpd_maxwait='', $commentlinks = [], $war, + $projects_file = 'UNDEF', $enable_melody = 'false', $melody_session = 'false', $mysql_password, @@ -77,6 +78,7 @@ class gerrit($virtual_hostname=$fqdn, $email_private_key, $replicate_github=false, $replicate_local=true, + $local_git_dir='/var/lib/git', $replication_targets=[], $testmode=false ) { @@ -165,6 +167,27 @@ class gerrit($virtual_hostname=$fqdn, } } + if ($projects_file != 'UNDEF') { + + file { '/home/gerrit2/projects.yaml': + owner => 'gerrit2', + group => 'gerrit2', + mode => 444, + ensure => 'present', + source => $projects_file, + replace => true, + } + + exec { "make_local_repos": + user => 'gerrit2', + command => "/usr/local/gerrit/scripts/make_local_repos.py $local_git_dir", + subscribe => File["/home/gerrit2/projects.yaml"], + refreshonly => true, + require => File["/home/gerrit2/projects.yaml"] + } + + } + # Gerrit sets these permissions in 'init'; don't fight them. file { '/home/gerrit2/review_site/etc/gerrit.config': owner => 'gerrit2', diff --git a/modules/gerrit/manifests/remotes.pp b/modules/gerrit/manifests/remotes.pp index da27242b10..88a12cefdc 100644 --- a/modules/gerrit/manifests/remotes.pp +++ b/modules/gerrit/manifests/remotes.pp @@ -1,18 +1,13 @@ -class gerrit::remotes($upstream_projects) { +class gerrit::remotes($ensure=present) { cron { "gerritfetchremotes": user => gerrit2, + ensure => $ensure, minute => "*/30", command => 'sleep $((RANDOM\%60+90)) && python /usr/local/gerrit/scripts/fetch_remotes.py', require => File['/usr/local/gerrit/scripts'], } file { '/home/gerrit2/remotes.config': - owner => 'root', - group => 'root', - mode => 444, - ensure => 'present', - content => template('gerrit/remotes.config.erb'), - replace => 'true', - require => User["gerrit2"] + ensure => absent } } diff --git a/modules/gerrit/templates/remotes.config.erb b/modules/gerrit/templates/remotes.config.erb deleted file mode 100644 index 2b9502f213..0000000000 --- a/modules/gerrit/templates/remotes.config.erb +++ /dev/null @@ -1,7 +0,0 @@ -# This file is managed by puppet. -# https://github.com/openstack/openstack-ci-puppet - -<% upstream_projects.each do |project| -%> -[project "<%= project['name'] %>"] -remote = <%= project['remote'] %> -<% end -%> diff --git a/modules/github/files/scripts/close_pull_requests.py b/modules/github/files/scripts/close_pull_requests.py index 202d2eb7a7..13f9e8994f 100755 --- a/modules/github/files/scripts/close_pull_requests.py +++ b/modules/github/files/scripts/close_pull_requests.py @@ -13,12 +13,12 @@ # License for the specific language governing permissions and limitations # under the License. -# Close Github pull requests with instructions to use Gerrit for -# code review. The list of projects is found in github.config -# and should look like: +# Github pull requests closer reads a project config file called projects.yaml +# It should look like: -# [project "GITHUB_PROJECT"] -# close_pull = true +# - project: PROJECT_NAME +# options: +# - close-pull # Github authentication information is read from github.secure.config, # which should look like: @@ -32,16 +32,16 @@ # [github] # oauth_token = GITHUB_OAUTH_TOKEN +import ConfigParser import github import os -import ConfigParser +import yaml import logging -import re logging.basicConfig(level=logging.ERROR) -GITHUB_CONFIG = os.environ.get('GITHUB_CONFIG', - '/home/gerrit2/github.config') +PROJECTS_YAML = os.environ.get('PROJECTS_YAML', + '/home/gerrit2/projects.yaml') GITHUB_SECURE_CONFIG = os.environ.get('GITHUB_SECURE_CONFIG', '/home/gerrit2/github.secure.config') @@ -52,12 +52,9 @@ MESSAGE = """Thank you for contributing to OpenStack! Please visit http://wiki.openstack.org/GerritWorkflow and follow the instructions there to upload your change to Gerrit. """ -PROJECT_RE = re.compile(r'^project\s+"(.*)"$') - secure_config = ConfigParser.ConfigParser() secure_config.read(GITHUB_SECURE_CONFIG) -config = ConfigParser.ConfigParser() -config.read(GITHUB_CONFIG) +config = yaml.load(open(PROJECTS_YAML)) if secure_config.has_option("github", "oauth_token"): ghub = github.Github(secure_config.get("github", "oauth_token")) @@ -67,15 +64,11 @@ else: orgs = ghub.get_user().get_orgs() orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs)) -for section in config.sections(): - # Each section looks like [project "openstack/project"] - m = PROJECT_RE.match(section) - if not m: continue - project = m.group(1) +for section in config: + project = section['project'] # Make sure we're supposed to close pull requests for this project: - if not (config.has_option(section, "close_pull") and - config.get(section, "close_pull").lower() == 'true'): + if 'options' not in section or 'close-pull' not in section['options']: continue # Find the project's repo diff --git a/modules/github/manifests/init.pp b/modules/github/manifests/init.pp index d4bc3253ef..24b055433c 100644 --- a/modules/github/manifests/init.pp +++ b/modules/github/manifests/init.pp @@ -32,13 +32,7 @@ class github ( } file { '/etc/github/github.config': - owner => 'root', - group => 'root', - mode => 444, - ensure => 'present', - content => template('github/github.config.erb'), - replace => 'true', - require => File['/etc/github'], + ensure => absent } file { '/etc/github/github.secure.config': diff --git a/modules/github/templates/github.config.erb b/modules/github/templates/github.config.erb deleted file mode 100644 index 2945ae9356..0000000000 --- a/modules/github/templates/github.config.erb +++ /dev/null @@ -1,7 +0,0 @@ -# This file is managed by puppet. -# https://github.com/openstack/openstack-ci-puppet - -<% projects.each do |project| -%> -[project "<%= project['name'] %>"] -close_pull = <%= project['close_pull'] %> -<% end -%> diff --git a/modules/openstack_project/files/review-dev.projects.yaml b/modules/openstack_project/files/review-dev.projects.yaml new file mode 100644 index 0000000000..fc52cf0987 --- /dev/null +++ b/modules/openstack_project/files/review-dev.projects.yaml @@ -0,0 +1,3 @@ +- project: gtest-org/test + options: + - close-pull diff --git a/modules/openstack_project/files/review.projects.yaml b/modules/openstack_project/files/review.projects.yaml new file mode 100644 index 0000000000..7d4caafd3b --- /dev/null +++ b/modules/openstack_project/files/review.projects.yaml @@ -0,0 +1,145 @@ +- project: heat-api/heat + options: + - close-pull +- project: openstack-ci/devstack-gate + options: + - close-pull +- project: openstack-ci/gerrit + options: + - close-pull + remote: https://gerrit.googlesource.com/gerrit +- project: openstack-ci/gerrit-verification-status-plugin + options: + - close-pull +- project: openstack-ci/gerritbot + options: + - close-pull +- project: openstack-ci/gerritlib + options: + - close-pull +- project: openstack-ci/git-review + options: + - close-pull +- project: openstack-ci/lodgeit + options: + - close-pull +- project: openstack-ci/meetbot + options: + - close-pull +- project: openstack-ci/pypi-mirror + options: + - close-pull +- project: openstack-ci/zuul + options: + - close-pull +- project: openstack-dev/devstack + options: + - close-pull +- project: openstack-dev/openstack-nose + options: + - close-pull +- project: openstack-dev/openstack-qa + options: + - close-pull +- project: openstack-dev/pbr + options: + - close-pull +- project: openstack-dev/sandbox + options: + - close-pull +- project: openstack/cinder + options: + - close-pull +- project: openstack/compute-api + options: + - close-pull +- project: openstack/glance + options: + - close-pull +- project: openstack/horizon + options: + - close-pull +- project: openstack/identity-api + options: + - close-pull +- project: openstack/image-api + options: + - close-pull +- project: openstack/keystone + options: + - close-pull +- project: openstack/melange + options: + - close-pull +- project: openstack/netconn-api + options: + - close-pull +- project: openstack/nova + options: + - close-pull +- project: openstack/object-api + options: + - close-pull +- project: openstack/openstack-chef + options: + - close-pull +- project: openstack/openstack-ci + options: + - close-pull +- project: openstack/openstack-ci-puppet + options: + - close-pull +- project: openstack/openstack-common + options: + - close-pull +- project: openstack/openstack-manuals + options: + - close-pull +- project: openstack/openstack-planet + options: + - close-pull +- project: openstack/openstack-puppet + options: + - close-pull +- project: openstack/python-cinderclient + options: + - close-pull +- project: openstack/python-glanceclient + options: + - close-pull +- project: openstack/python-keystoneclient + options: + - close-pull +- project: openstack/python-melangeclient + options: + - close-pull +- project: openstack/python-novaclient + options: + - close-pull +- project: openstack/python-openstackclient + options: + - close-pull +- project: openstack/python-quantumclient + options: + - close-pull +- project: openstack/python-swiftclient + options: + - close-pull +- project: openstack/quantum + options: + - close-pull +- project: openstack/swift + options: + - close-pull +- project: openstack/tempest + options: + - close-pull +- project: stackforge/MRaaS + options: + - close-pull +- project: stackforge/ceilometer + options: + - close-pull +- project: stackforge/reddwarf + options: + - close-pull diff --git a/modules/openstack_project/manifests/gerrit.pp b/modules/openstack_project/manifests/gerrit.pp index 6f6e834bd0..cf04ab1159 100644 --- a/modules/openstack_project/manifests/gerrit.pp +++ b/modules/openstack_project/manifests/gerrit.pp @@ -23,7 +23,7 @@ class openstack_project::gerrit ( $war, $script_user='update', $script_key_file='/home/gerrit2/.ssh/id_rsa', - $github_projects = [], + $projects_file='UNDEF', $github_username, $github_oauth_token, $mysql_password, @@ -70,6 +70,7 @@ class openstack_project::gerrit ( mysql_password => $mysql_password, mysql_root_password => $mysql_root_password, email_private_key => $email_private_key, + projects_file => $projects_file, replicate_github => true, testmode => $testmode, require => Class[openstack_project::server], @@ -80,9 +81,9 @@ class openstack_project::gerrit ( script_key_file => $script_key_file, } class { 'github': - projects => $github_projects, username => $github_username, oauth_token => $github_oauth_token, + require => Class['::gerrit'] } } diff --git a/modules/openstack_project/manifests/init.pp b/modules/openstack_project/manifests/init.pp index 8bc492a8de..572149720d 100644 --- a/modules/openstack_project/manifests/init.pp +++ b/modules/openstack_project/manifests/init.pp @@ -9,121 +9,4 @@ class openstack_project { 'devananda.vdv@gmail.com', 'clark.boylan@gmail.com' ] - - $project_list = [ { - name => 'openstack/keystone', - close_pull => 'true' - }, { - name => 'openstack/glance', - close_pull => 'true' - }, { - name => 'openstack/swift', - close_pull => 'true' - }, { - name => 'openstack/nova', - close_pull => 'true' - }, { - name => 'openstack/horizon', - close_pull => 'true' - }, { - name => 'openstack/quantum', - close_pull => 'true' - }, { - name => 'openstack/melange', - close_pull => 'true' - }, { - name => 'openstack/tempest', - close_pull => 'true' - }, { - name => 'openstack/openstack-ci', - close_pull => 'true' - }, { - name => 'openstack/openstack-ci-puppet', - close_pull => 'true' - }, { - name => 'openstack/openstack-puppet', - close_pull => 'true' - }, { - name => 'openstack/openstack-chef', - close_pull => 'true' - }, { - name => 'openstack/openstack-manuals', - close_pull => 'true' - }, { - name => 'openstack/compute-api', - close_pull => 'true' - }, { - name => 'openstack/image-api', - close_pull => 'true' - }, { - name => 'openstack/identity-api', - close_pull => 'true' - }, { - name => 'openstack/object-api', - close_pull => 'true' - }, { - name => 'openstack/netconn-api', - close_pull => 'true' - }, { - name => 'openstack-dev/devstack', - close_pull => 'true' - }, { - name => 'openstack-dev/openstack-qa', - close_pull => 'true' - }, { - name => 'openstack-dev/pbr', - close_pull => 'true' - }, { - name => 'openstack/python-novaclient', - close_pull => 'true' - }, { - name => 'openstack/python-glanceclient', - close_pull => 'true' - }, { - name => 'openstack-ci/git-review', - close_pull => 'true' - }, { - name => 'openstack-ci/lodgeit', - close_pull => 'true' - }, { - name => 'openstack-ci/meetbot', - close_pull => 'true' - }, { - name => 'openstack-ci/zuul', - close_pull => 'true' - }, { - name => 'openstack-ci/pypi-mirror', - close_pull => 'true' - }, { - name => 'openstack/openstack-common', - close_pull => 'true' - }, { - name => 'openstack/cinder', - close_pull => 'true' - }, { - name => 'openstack/python-openstackclient', - close_pull => 'true' - }, { - name => 'openstack-dev/openstack-nose', - close_pull => 'true' - }, { - name => 'openstack/python-cinderclient', - close_pull => 'true' - }, { - name => 'openstack/python-swiftclient', - close_pull => 'true' - }, { - name => 'stackforge/MRaaS', - close_pull => 'true' - }, { - name => 'stackforge/reddwarf', - close_pull => 'true' - }, { - name => 'stackforge/ceilometer', - close_pull => 'true' - }, { - name => 'heat-api/heat', - close_pull => 'true' - } - ] } diff --git a/modules/openstack_project/manifests/review.pp b/modules/openstack_project/manifests/review.pp index 8f323299d4..1bac91c75f 100644 --- a/modules/openstack_project/manifests/review.pp +++ b/modules/openstack_project/manifests/review.pp @@ -46,7 +46,7 @@ class openstack_project::review( war => 'http://tarballs.openstack.org/ci/gerrit-2.4.2-11-gb5a28fb.war', script_user => 'launchpadsync', script_key_file => '/home/gerrit2/.ssh/launchpadsync_rsa', - github_projects => $openstack_project::project_list, + projects_file => 'puppet:///openstack_project/review.projects.yaml', github_username => 'openstack-gerrit', github_oauth_token => $github_oauth_token, mysql_password => $mysql_password, @@ -60,10 +60,5 @@ class openstack_project::review( user => 'gerritbot', virtual_hostname => $fqdn } - class { 'gerrit::remotes': - upstream_projects => [ { - name => 'openstack-ci/gerrit', - remote => 'https://gerrit.googlesource.com/gerrit' - } ], - } + include gerrit::remotes } diff --git a/modules/openstack_project/manifests/review_dev.pp b/modules/openstack_project/manifests/review_dev.pp index 3d1d6714e8..9ecd48dfb7 100644 --- a/modules/openstack_project/manifests/review_dev.pp +++ b/modules/openstack_project/manifests/review_dev.pp @@ -11,10 +11,7 @@ class openstack_project::review_dev( ssl_chain_file => '', email => "review-dev@openstack.org", war => 'http://tarballs.openstack.org/ci/gerrit-2.4.2-11-gb5a28fb.war', - github_projects => [ { - name => 'gtest-org/test', - close_pull => 'true' - } ], + projects_file => 'puppet:///openstack_project/review-dev.projects.yaml', github_username => 'openstack-gerrit-dev', github_oauth_token => $github_oauth_token, mysql_password => $mysql_password,