diff --git a/modules/gerrit/files/scripts/fetch_remotes.py b/modules/gerrit/files/scripts/fetch_remotes.py index 776188e765..4dda133f3d 100755 --- a/modules/gerrit/files/scripts/fetch_remotes.py +++ b/modules/gerrit/files/scripts/fetch_remotes.py @@ -13,18 +13,19 @@ # License for the specific language governing permissions and limitations # under the License. -# Ensure that the specified remote exists in the repo and pull revisions -# from upstream +# Fetch remotes reads a project config file called projects.yaml +# It should look like: + +# - project: PROJECT_NAME +# options: +# - remote: https://gerrit.googlesource.com/gerrit -# [project "UPSTREAM_PROJECT"] -# remote = https://gerrit.googlesource.com/gerrit -import ConfigParser import logging import os -import re import subprocess import shlex +import yaml def run_command(cmd, status=False, env={}): cmd_list = shlex.split(str(cmd)) @@ -46,29 +47,22 @@ logging.basicConfig(level=logging.ERROR) REPO_ROOT = os.environ.get('REPO_ROOT', '/home/gerrit2/review_site/git') -REMOTES_CONFIG = os.environ.get('REMOTES_CONFIG', - '/home/gerrit2/remotes.config') +PROJECTS_YAML = os.environ.get('PROJECTS_YAML', + '/home/gerrit2/projects.yaml') -PROJECT_RE = re.compile(r'^project\s+"(.*)"$') +config = yaml.load(open(PROJECTS_YAML)) -config = ConfigParser.ConfigParser() -config.read(REMOTES_CONFIG) +for section in config: + project = section['project'] - -for section in config.sections(): - # Each section looks like [project "openstack/project"] - m = PROJECT_RE.match(section) - if not m: + if 'remote' not in section: continue - project = m.group(1) + project_git = "%s.git" % project os.chdir(os.path.join(REPO_ROOT, project_git)) - if not (config.has_option(section, "remote")): - continue - # Make sure that the specified remote exists - remote_url = config.get(section, "remote") + remote_url = section['remote'] # We could check if it exists first, but we're ignoring output anyway # So just try to make it, and it'll either make a new one or do nothing run_command("git remote add -f upstream %s" % remote_url) diff --git a/modules/gerrit/files/scripts/make_local_repos.py b/modules/gerrit/files/scripts/make_local_repos.py new file mode 100755 index 0000000000..f598446168 --- /dev/null +++ b/modules/gerrit/files/scripts/make_local_repos.py @@ -0,0 +1,67 @@ +#! /usr/bin/env python +# Copyright (C) 2011 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Make local repos reads a project config file called projects.yaml +# It should look like: + +# - project: PROJECT_NAME +# options: +# - close-pull +# remote: https://gerrit.googlesource.com/gerrit + +# TODO: add support for +# ssh -p 29418 localhost gerrit -name create-project PROJECT + +import logging +import os +import subprocess +import sys +import shlex +import yaml + +def run_command(cmd, status=False, env={}): + cmd_list = shlex.split(str(cmd)) + newenv = os.environ + newenv.update(env) + p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, env=newenv) + (out, nothing) = p.communicate() + if status: + return (p.returncode, out.strip()) + return out.strip() + + +def run_command_status(cmd, env={}): + return run_command(cmd, True, env) + + +logging.basicConfig(level=logging.ERROR) + +REPO_ROOT = sys.argv[1] +PROJECTS_YAML = os.environ.get('PROJECTS_YAML', + '/home/gerrit2/projects.yaml') + +config = yaml.load(open(PROJECTS_YAML)) + +for section in config: + project = section['project'] + + project_git = "%s.git" % project + project_dir = os.path.join(REPO_ROOT, project_git) + + if os.path.exists(project_dir): + continue + + run_command("git --bare init --shared=group %s" % project_dir) diff --git a/modules/gerrit/manifests/init.pp b/modules/gerrit/manifests/init.pp index 122a5f8401..d93b42c39d 100644 --- a/modules/gerrit/manifests/init.pp +++ b/modules/gerrit/manifests/init.pp @@ -70,6 +70,7 @@ class gerrit($virtual_hostname=$fqdn, $httpd_maxwait='', $commentlinks = [], $war, + $projects_file = 'UNDEF', $enable_melody = 'false', $melody_session = 'false', $mysql_password, @@ -77,6 +78,7 @@ class gerrit($virtual_hostname=$fqdn, $email_private_key, $replicate_github=false, $replicate_local=true, + $local_git_dir='/var/lib/git', $replication_targets=[], $testmode=false ) { @@ -165,6 +167,27 @@ class gerrit($virtual_hostname=$fqdn, } } + if ($projects_file != 'UNDEF') { + + file { '/home/gerrit2/projects.yaml': + owner => 'gerrit2', + group => 'gerrit2', + mode => 444, + ensure => 'present', + source => $projects_file, + replace => true, + } + + exec { "make_local_repos": + user => 'gerrit2', + command => "/usr/local/gerrit/scripts/make_local_repos.py $local_git_dir", + subscribe => File["/home/gerrit2/projects.yaml"], + refreshonly => true, + require => File["/home/gerrit2/projects.yaml"] + } + + } + # Gerrit sets these permissions in 'init'; don't fight them. file { '/home/gerrit2/review_site/etc/gerrit.config': owner => 'gerrit2', diff --git a/modules/gerrit/manifests/remotes.pp b/modules/gerrit/manifests/remotes.pp index da27242b10..88a12cefdc 100644 --- a/modules/gerrit/manifests/remotes.pp +++ b/modules/gerrit/manifests/remotes.pp @@ -1,18 +1,13 @@ -class gerrit::remotes($upstream_projects) { +class gerrit::remotes($ensure=present) { cron { "gerritfetchremotes": user => gerrit2, + ensure => $ensure, minute => "*/30", command => 'sleep $((RANDOM\%60+90)) && python /usr/local/gerrit/scripts/fetch_remotes.py', require => File['/usr/local/gerrit/scripts'], } file { '/home/gerrit2/remotes.config': - owner => 'root', - group => 'root', - mode => 444, - ensure => 'present', - content => template('gerrit/remotes.config.erb'), - replace => 'true', - require => User["gerrit2"] + ensure => absent } } diff --git a/modules/gerrit/templates/remotes.config.erb b/modules/gerrit/templates/remotes.config.erb deleted file mode 100644 index 2b9502f213..0000000000 --- a/modules/gerrit/templates/remotes.config.erb +++ /dev/null @@ -1,7 +0,0 @@ -# This file is managed by puppet. -# https://github.com/openstack/openstack-ci-puppet - -<% upstream_projects.each do |project| -%> -[project "<%= project['name'] %>"] -remote = <%= project['remote'] %> -<% end -%> diff --git a/modules/github/files/scripts/close_pull_requests.py b/modules/github/files/scripts/close_pull_requests.py index 202d2eb7a7..13f9e8994f 100755 --- a/modules/github/files/scripts/close_pull_requests.py +++ b/modules/github/files/scripts/close_pull_requests.py @@ -13,12 +13,12 @@ # License for the specific language governing permissions and limitations # under the License. -# Close Github pull requests with instructions to use Gerrit for -# code review. The list of projects is found in github.config -# and should look like: +# Github pull requests closer reads a project config file called projects.yaml +# It should look like: -# [project "GITHUB_PROJECT"] -# close_pull = true +# - project: PROJECT_NAME +# options: +# - close-pull # Github authentication information is read from github.secure.config, # which should look like: @@ -32,16 +32,16 @@ # [github] # oauth_token = GITHUB_OAUTH_TOKEN +import ConfigParser import github import os -import ConfigParser +import yaml import logging -import re logging.basicConfig(level=logging.ERROR) -GITHUB_CONFIG = os.environ.get('GITHUB_CONFIG', - '/home/gerrit2/github.config') +PROJECTS_YAML = os.environ.get('PROJECTS_YAML', + '/home/gerrit2/projects.yaml') GITHUB_SECURE_CONFIG = os.environ.get('GITHUB_SECURE_CONFIG', '/home/gerrit2/github.secure.config') @@ -52,12 +52,9 @@ MESSAGE = """Thank you for contributing to OpenStack! Please visit http://wiki.openstack.org/GerritWorkflow and follow the instructions there to upload your change to Gerrit. """ -PROJECT_RE = re.compile(r'^project\s+"(.*)"$') - secure_config = ConfigParser.ConfigParser() secure_config.read(GITHUB_SECURE_CONFIG) -config = ConfigParser.ConfigParser() -config.read(GITHUB_CONFIG) +config = yaml.load(open(PROJECTS_YAML)) if secure_config.has_option("github", "oauth_token"): ghub = github.Github(secure_config.get("github", "oauth_token")) @@ -67,15 +64,11 @@ else: orgs = ghub.get_user().get_orgs() orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs)) -for section in config.sections(): - # Each section looks like [project "openstack/project"] - m = PROJECT_RE.match(section) - if not m: continue - project = m.group(1) +for section in config: + project = section['project'] # Make sure we're supposed to close pull requests for this project: - if not (config.has_option(section, "close_pull") and - config.get(section, "close_pull").lower() == 'true'): + if 'options' not in section or 'close-pull' not in section['options']: continue # Find the project's repo diff --git a/modules/github/manifests/init.pp b/modules/github/manifests/init.pp index d4bc3253ef..24b055433c 100644 --- a/modules/github/manifests/init.pp +++ b/modules/github/manifests/init.pp @@ -32,13 +32,7 @@ class github ( } file { '/etc/github/github.config': - owner => 'root', - group => 'root', - mode => 444, - ensure => 'present', - content => template('github/github.config.erb'), - replace => 'true', - require => File['/etc/github'], + ensure => absent } file { '/etc/github/github.secure.config': diff --git a/modules/github/templates/github.config.erb b/modules/github/templates/github.config.erb deleted file mode 100644 index 2945ae9356..0000000000 --- a/modules/github/templates/github.config.erb +++ /dev/null @@ -1,7 +0,0 @@ -# This file is managed by puppet. -# https://github.com/openstack/openstack-ci-puppet - -<% projects.each do |project| -%> -[project "<%= project['name'] %>"] -close_pull = <%= project['close_pull'] %> -<% end -%> diff --git a/modules/openstack_project/files/review-dev.projects.yaml b/modules/openstack_project/files/review-dev.projects.yaml new file mode 100644 index 0000000000..fc52cf0987 --- /dev/null +++ b/modules/openstack_project/files/review-dev.projects.yaml @@ -0,0 +1,3 @@ +- project: gtest-org/test + options: + - close-pull diff --git a/modules/openstack_project/files/review.projects.yaml b/modules/openstack_project/files/review.projects.yaml new file mode 100644 index 0000000000..7d4caafd3b --- /dev/null +++ b/modules/openstack_project/files/review.projects.yaml @@ -0,0 +1,145 @@ +- project: heat-api/heat + options: + - close-pull +- project: openstack-ci/devstack-gate + options: + - close-pull +- project: openstack-ci/gerrit + options: + - close-pull + remote: https://gerrit.googlesource.com/gerrit +- project: openstack-ci/gerrit-verification-status-plugin + options: + - close-pull +- project: openstack-ci/gerritbot + options: + - close-pull +- project: openstack-ci/gerritlib + options: + - close-pull +- project: openstack-ci/git-review + options: + - close-pull +- project: openstack-ci/lodgeit + options: + - close-pull +- project: openstack-ci/meetbot + options: + - close-pull +- project: openstack-ci/pypi-mirror + options: + - close-pull +- project: openstack-ci/zuul + options: + - close-pull +- project: openstack-dev/devstack + options: + - close-pull +- project: openstack-dev/openstack-nose + options: + - close-pull +- project: openstack-dev/openstack-qa + options: + - close-pull +- project: openstack-dev/pbr + options: + - close-pull +- project: openstack-dev/sandbox + options: + - close-pull +- project: openstack/cinder + options: + - close-pull +- project: openstack/compute-api + options: + - close-pull +- project: openstack/glance + options: + - close-pull +- project: openstack/horizon + options: + - close-pull +- project: openstack/identity-api + options: + - close-pull +- project: openstack/image-api + options: + - close-pull +- project: openstack/keystone + options: + - close-pull +- project: openstack/melange + options: + - close-pull +- project: openstack/netconn-api + options: + - close-pull +- project: openstack/nova + options: + - close-pull +- project: openstack/object-api + options: + - close-pull +- project: openstack/openstack-chef + options: + - close-pull +- project: openstack/openstack-ci + options: + - close-pull +- project: openstack/openstack-ci-puppet + options: + - close-pull +- project: openstack/openstack-common + options: + - close-pull +- project: openstack/openstack-manuals + options: + - close-pull +- project: openstack/openstack-planet + options: + - close-pull +- project: openstack/openstack-puppet + options: + - close-pull +- project: openstack/python-cinderclient + options: + - close-pull +- project: openstack/python-glanceclient + options: + - close-pull +- project: openstack/python-keystoneclient + options: + - close-pull +- project: openstack/python-melangeclient + options: + - close-pull +- project: openstack/python-novaclient + options: + - close-pull +- project: openstack/python-openstackclient + options: + - close-pull +- project: openstack/python-quantumclient + options: + - close-pull +- project: openstack/python-swiftclient + options: + - close-pull +- project: openstack/quantum + options: + - close-pull +- project: openstack/swift + options: + - close-pull +- project: openstack/tempest + options: + - close-pull +- project: stackforge/MRaaS + options: + - close-pull +- project: stackforge/ceilometer + options: + - close-pull +- project: stackforge/reddwarf + options: + - close-pull diff --git a/modules/openstack_project/manifests/gerrit.pp b/modules/openstack_project/manifests/gerrit.pp index 6f6e834bd0..cf04ab1159 100644 --- a/modules/openstack_project/manifests/gerrit.pp +++ b/modules/openstack_project/manifests/gerrit.pp @@ -23,7 +23,7 @@ class openstack_project::gerrit ( $war, $script_user='update', $script_key_file='/home/gerrit2/.ssh/id_rsa', - $github_projects = [], + $projects_file='UNDEF', $github_username, $github_oauth_token, $mysql_password, @@ -70,6 +70,7 @@ class openstack_project::gerrit ( mysql_password => $mysql_password, mysql_root_password => $mysql_root_password, email_private_key => $email_private_key, + projects_file => $projects_file, replicate_github => true, testmode => $testmode, require => Class[openstack_project::server], @@ -80,9 +81,9 @@ class openstack_project::gerrit ( script_key_file => $script_key_file, } class { 'github': - projects => $github_projects, username => $github_username, oauth_token => $github_oauth_token, + require => Class['::gerrit'] } } diff --git a/modules/openstack_project/manifests/init.pp b/modules/openstack_project/manifests/init.pp index 8bc492a8de..572149720d 100644 --- a/modules/openstack_project/manifests/init.pp +++ b/modules/openstack_project/manifests/init.pp @@ -9,121 +9,4 @@ class openstack_project { 'devananda.vdv@gmail.com', 'clark.boylan@gmail.com' ] - - $project_list = [ { - name => 'openstack/keystone', - close_pull => 'true' - }, { - name => 'openstack/glance', - close_pull => 'true' - }, { - name => 'openstack/swift', - close_pull => 'true' - }, { - name => 'openstack/nova', - close_pull => 'true' - }, { - name => 'openstack/horizon', - close_pull => 'true' - }, { - name => 'openstack/quantum', - close_pull => 'true' - }, { - name => 'openstack/melange', - close_pull => 'true' - }, { - name => 'openstack/tempest', - close_pull => 'true' - }, { - name => 'openstack/openstack-ci', - close_pull => 'true' - }, { - name => 'openstack/openstack-ci-puppet', - close_pull => 'true' - }, { - name => 'openstack/openstack-puppet', - close_pull => 'true' - }, { - name => 'openstack/openstack-chef', - close_pull => 'true' - }, { - name => 'openstack/openstack-manuals', - close_pull => 'true' - }, { - name => 'openstack/compute-api', - close_pull => 'true' - }, { - name => 'openstack/image-api', - close_pull => 'true' - }, { - name => 'openstack/identity-api', - close_pull => 'true' - }, { - name => 'openstack/object-api', - close_pull => 'true' - }, { - name => 'openstack/netconn-api', - close_pull => 'true' - }, { - name => 'openstack-dev/devstack', - close_pull => 'true' - }, { - name => 'openstack-dev/openstack-qa', - close_pull => 'true' - }, { - name => 'openstack-dev/pbr', - close_pull => 'true' - }, { - name => 'openstack/python-novaclient', - close_pull => 'true' - }, { - name => 'openstack/python-glanceclient', - close_pull => 'true' - }, { - name => 'openstack-ci/git-review', - close_pull => 'true' - }, { - name => 'openstack-ci/lodgeit', - close_pull => 'true' - }, { - name => 'openstack-ci/meetbot', - close_pull => 'true' - }, { - name => 'openstack-ci/zuul', - close_pull => 'true' - }, { - name => 'openstack-ci/pypi-mirror', - close_pull => 'true' - }, { - name => 'openstack/openstack-common', - close_pull => 'true' - }, { - name => 'openstack/cinder', - close_pull => 'true' - }, { - name => 'openstack/python-openstackclient', - close_pull => 'true' - }, { - name => 'openstack-dev/openstack-nose', - close_pull => 'true' - }, { - name => 'openstack/python-cinderclient', - close_pull => 'true' - }, { - name => 'openstack/python-swiftclient', - close_pull => 'true' - }, { - name => 'stackforge/MRaaS', - close_pull => 'true' - }, { - name => 'stackforge/reddwarf', - close_pull => 'true' - }, { - name => 'stackforge/ceilometer', - close_pull => 'true' - }, { - name => 'heat-api/heat', - close_pull => 'true' - } - ] } diff --git a/modules/openstack_project/manifests/review.pp b/modules/openstack_project/manifests/review.pp index 8f323299d4..1bac91c75f 100644 --- a/modules/openstack_project/manifests/review.pp +++ b/modules/openstack_project/manifests/review.pp @@ -46,7 +46,7 @@ class openstack_project::review( war => 'http://tarballs.openstack.org/ci/gerrit-2.4.2-11-gb5a28fb.war', script_user => 'launchpadsync', script_key_file => '/home/gerrit2/.ssh/launchpadsync_rsa', - github_projects => $openstack_project::project_list, + projects_file => 'puppet:///openstack_project/review.projects.yaml', github_username => 'openstack-gerrit', github_oauth_token => $github_oauth_token, mysql_password => $mysql_password, @@ -60,10 +60,5 @@ class openstack_project::review( user => 'gerritbot', virtual_hostname => $fqdn } - class { 'gerrit::remotes': - upstream_projects => [ { - name => 'openstack-ci/gerrit', - remote => 'https://gerrit.googlesource.com/gerrit' - } ], - } + include gerrit::remotes } diff --git a/modules/openstack_project/manifests/review_dev.pp b/modules/openstack_project/manifests/review_dev.pp index 3d1d6714e8..9ecd48dfb7 100644 --- a/modules/openstack_project/manifests/review_dev.pp +++ b/modules/openstack_project/manifests/review_dev.pp @@ -11,10 +11,7 @@ class openstack_project::review_dev( ssl_chain_file => '', email => "review-dev@openstack.org", war => 'http://tarballs.openstack.org/ci/gerrit-2.4.2-11-gb5a28fb.war', - github_projects => [ { - name => 'gtest-org/test', - close_pull => 'true' - } ], + projects_file => 'puppet:///openstack_project/review-dev.projects.yaml', github_username => 'openstack-gerrit-dev', github_oauth_token => $github_oauth_token, mysql_password => $mysql_password,