opendev/system-config
Code Issues Proposed changes

Move Zuul to its own server.

Browse Source 
Remove zuul from jenkins server.
Remove transitional zuul config.
Add an extra argument to gerrit-git-prep for the git site name, so that it
doesn't have to be the same as the review site.
Serve zuul git repos via apache.

Change-Id: I342032bcedf317c0f48ff89b773546eb2ccd08f8
Reviewed-on: https://review.openstack.org/17949
Tested-by: Jenkins
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Approved: James E. Blair <corvus@inaugust.com>
Reviewed-by: James E. Blair <corvus@inaugust.com>
Tested-by: James E. Blair <corvus@inaugust.com>
This commit is contained in:
James E. Blair 2012-12-02 16:32:10 +00:00 committed by James E. Blair
parent 3e3abe433b
commit 5b9461b5e1
8 changed files with 126 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
manifests/site.pp
View File

@ -71,14 +71,6 @@ node 'jenkins.openstack.org' {
ssl_chain_file_contents => hiera('jenkins_ssl_chain_file_contents'),
sysadmins => hiera('sysadmins'),
}
class { 'openstack_project::zuul':
jenkins_url => "https://${::fqdn}",
jenkins_user => 'hudson-openstack',
jenkins_apikey => hiera('zuul_jenkins_apikey'),
gerrit_server => 'review.openstack.org',
gerrit_user => 'jenkins',
url_pattern => 'http://logs.openstack.org/{change.number}/{change.patchset}/{pipeline.name}/{job.name}/{build.number}',
}
}
node 'jenkins-dev.openstack.org' {
@ -193,14 +185,15 @@ node 'static.openstack.org' {
node 'zuul.openstack.org' {
class { 'openstack_project::zuul':
jenkins_host => 'jenkins.openstack.org',
jenkins_url => 'https://jenkins.openstack.org',
jenkins_user => 'hudson-openstack',
jenkins_apikey => hiera('zuul_jenkins_apikey'),
gerrit_server => 'review.openstack.org',
gerrit_user => 'jenkins',
url_pattern => 'http://logs.openstack.org/{change.number}/{change.patchset}/{pipeline.name}/{job.name}/{build.number}',
sysadmins => hiera('sysadmins'),
jenkins_host => 'jenkins.openstack.org',
jenkins_url => 'https://jenkins.openstack.org',
jenkins_user => 'hudson-openstack',
jenkins_apikey => hiera('zuul_jenkins_apikey'),
gerrit_server => 'review.openstack.org',
gerrit_user => 'jenkins',
zuul_ssh_private_key => hiera('jenkins_ssh_private_key_contents'),
url_pattern => 'http://logs.openstack.org/{change.number}/{change.patchset}/{pipeline.name}/{job.name}/{build.number}',
sysadmins => hiera('sysadmins'),
}
}

23
modules/jenkins/files/slave_scripts/gerrit-git-prep.sh
View File

@ -10,10 +10,17 @@
# GERRIT_CHANGES="gtest-org/test:master:refs/changes/21/421/1"
# GERRIT_CHANGES=""
SITE=$1
if [ -z "$SITE" ]
REVIEW_SITE=$1
if [ -z "$REVIEW_SITE" ]
then
echo "The site name (eg 'review.openstack.org') must be the first argument."
echo "The git site name (eg 'https://review.openstack.org') must be the first argument."
exit 1
fi
GIT_SITE=$2
if [ -z "$GIT_SITE" ]
then
echo "The git site name (eg 'http://zuul.openstack.org') must be the second argument."
exit 1
fi
@ -26,13 +33,13 @@ fi
if [ ! -z "$GERRIT_CHANGES" ]
then
CHANGE_NUMBER=`echo $GERRIT_CHANGES|grep -Po ".*/\K\d+(?=/\d+)"`
echo "Triggered by: https://$SITE/$CHANGE_NUMBER"
echo "Triggered by: $REVIEW_SITE/$CHANGE_NUMBER"
fi
if [ ! -z "$GERRIT_REFSPEC" ]
then
CHANGE_NUMBER=`echo $GERRIT_REFSPEC|grep -Po ".*/\K\d+(?=/\d+)"`
echo "Triggered by: https://$SITE/$CHANGE_NUMBER"
echo "Triggered by: $REVIEW_SITE/$CHANGE_NUMBER"
fi
function merge_change {
@ -41,10 +48,10 @@ function merge_change {
MAX_ATTEMPTS=${3:-3}
COUNT=0
until git fetch https://$SITE/p/$PROJECT $REFSPEC
until git fetch $GIT_SITE/p/$PROJECT $REFSPEC
do
COUNT=$(($COUNT + 1))
logger -p user.warning -t 'gerrit-git-prep' FAILED: git fetch https://$SITE/p/$PROJECT $REFSPEC COUNT: $COUNT
logger -p user.warning -t 'gerrit-git-prep' FAILED: git fetch $GIT_SITE/p/$PROJECT $REFSPEC COUNT: $COUNT
if [ $COUNT -eq $MAX_ATTEMPTS ]
then
break
@ -97,7 +104,7 @@ function merge_changes {
set -x
if [[ ! -e .git ]]
then
git clone https://$SITE/p/$GERRIT_PROJECT .
git clone $GIT_SITE/p/$GERRIT_PROJECT .
fi
git remote update || git remote update # attempt to work around bug #925790
git reset --hard

4
modules/openstack_project/files/jenkins_job_builder/config/macros.yaml
View File

@ -1,7 +1,7 @@
- builder:
name: gerrit-git-prep
builders:
- shell: "/usr/local/jenkins/slave_scripts/gerrit-git-prep.sh review.openstack.org"
- shell: "/usr/local/jenkins/slave_scripts/gerrit-git-prep.sh https://review.openstack.org http://zuul.openstack.org"
- builder:
name: coverage
@ -109,7 +109,7 @@
- shell: |
#!/bin/bash -xe
if [[ ! -e devstack-gate ]]; then
git clone https://review.openstack.org/p/openstack-ci/devstack-gate
git clone http://zuul.openstack.org/p/openstack-ci/devstack-gate
else
cd devstack-gate
git remote update

29
modules/openstack_project/manifests/zuul.pp
View File

@ -1,35 +1,36 @@
# == Class: openstack_project::zuul
#
class openstack_project::zuul(
$vhost_name = $::fqdn,
$jenkins_host = '',
$jenkins_url = '',
$jenkins_user = '',
$jenkins_apikey = '',
$gerrit_server = '',
$gerrit_user = '',
$zuul_ssh_private_key = '',
$url_pattern = '',
$sysadmins = []
) {
$rules = [ "-m state --state NEW -m tcp -p tcp --dport 8001 -s ${jenkins_host} -j ACCEPT" ]
# TODO: This is temporary to handle the transition to a standalone server
if ($sysadmins != []) {
class { 'openstack_project::server':
iptables_public_tcp_ports => [80, 443],
iptables_rules4 => $rules,
sysadmins => $sysadmins,
}
class { 'openstack_project::server':
iptables_public_tcp_ports => [80],
iptables_rules4 => $rules,
sysadmins => $sysadmins,
}
class { '::zuul':
jenkins_server => $jenkins_url,
jenkins_user => $jenkins_user,
jenkins_apikey => $jenkins_apikey,
gerrit_server => $gerrit_server,
gerrit_user => $gerrit_user,
url_pattern => $url_pattern,
push_change_refs => true
vhost_name => $vhost_name,
jenkins_server => $jenkins_url,
jenkins_user => $jenkins_user,
jenkins_apikey => $jenkins_apikey,
gerrit_server => $gerrit_server,
gerrit_user => $gerrit_user,
zuul_ssh_private_key => $zuul_ssh_private_key,
url_pattern => $url_pattern,
push_change_refs => true
}
file { '/etc/zuul/layout.yaml':

2
modules/zuul/files/zuul.init
View File

@ -18,7 +18,7 @@ NAME=zuul
DAEMON=/usr/local/bin/zuul-server
PIDFILE=/var/run/$NAME/$NAME.pid
SCRIPTNAME=/etc/init.d/$NAME
USER=jenkins
USER=zuul
# Exit if the package is not installed
[ -x "$DAEMON" ] || exit 0

75
modules/zuul/manifests/init.pp
View File

@ -1,16 +1,21 @@
# == Class: zuul
#
class zuul (
$vhost_name = $::fqdn,
$serveradmin = "webmaster@${::fqdn}",
$jenkins_server = '',
$jenkins_user = '',
$jenkins_apikey = '',
$gerrit_server = '',
$gerrit_user = '',
$zuul_ssh_private_key = '',
$url_pattern = '',
$status_url = "https://${::fqdn}/zuul/status",
$status_url = "https://${::fqdn}/",
$git_source_repo = 'https://github.com/openstack-ci/zuul.git',
$push_change_refs = false
) {
include apache
$packages = [
'python-webob',
'python-daemon',
@ -22,6 +27,19 @@ class zuul (
ensure => present,
}
user { 'zuul':
ensure => present,
home => '/home/zuul',
shell => '/bin/bash',
gid => 'zuul',
managehome => true,
require => Group['zuul'],
}
group { 'zuul':
ensure => present,
}
# A lot of things need yaml, be conservative requiring this package to avoid
# conflicts with other modules.
if ! defined(Package['python-yaml']) {
@ -71,40 +89,56 @@ class zuul (
# at some point, but that still has some problems.
file { '/etc/zuul/zuul.conf':
ensure => present,
owner => 'jenkins',
owner => 'zuul',
mode => '0400',
content => template('zuul/zuul.conf.erb'),
require => [
File['/etc/zuul'],
Package['jenkins'],
User['zuul'],
],
}
file { '/var/log/zuul':
ensure => directory,
owner => 'jenkins',
require => Package['jenkins'],
owner => 'zuul',
require => User['zuul'],
}
file { '/var/run/zuul':
ensure => directory,
owner => 'jenkins',
require => Package['jenkins'],
owner => 'zuul',
require => User['zuul'],
}
file { '/var/lib/zuul':
ensure => directory,
owner => 'jenkins',
require => Package['jenkins'],
owner => 'zuul',
require => User['zuul'],
}
file { '/var/lib/zuul/git':
ensure => directory,
owner => 'jenkins',
require => Package['jenkins'],
owner => 'zuul',
require => User['zuul'],
}
file { '/etc/init.d/zuul/':
file { '/var/lib/zuul/ssh':
ensure => directory,
owner => 'zuul',
group => 'zuul',
mode => '0500',
require => File['/var/lib/zuul'],
}
file { '/var/lib/zuul/ssh/id_rsa':
owner => 'zuul',
group => 'zuul',
mode => '0400',
require => File['/var/lib/zuul/ssh'],
content => $zuul_ssh_private_key,
}
file { '/etc/init.d/zuul':
ensure => present,
owner => 'root',
group => 'root',
@ -124,4 +158,21 @@ class zuul (
hasrestart => true,
require => File['/etc/init.d/zuul'],
}
apache::vhost { $vhost_name:
port => 443,
docroot => 'MEANINGLESS ARGUMENT',
priority => '50',
template => 'zuul/zuul.vhost.erb',
}
a2mod { 'rewrite':
ensure => present,
}
a2mod { 'proxy':
ensure => present,
}
a2mod { 'proxy_http':
ensure => present,
}
}

2
modules/zuul/templates/zuul.conf.erb
View File

@ -6,7 +6,7 @@ apikey=<%= jenkins_apikey %>
[gerrit]
server=<%= gerrit_server %>
user=<%= gerrit_user %>
sshkey=/var/lib/jenkins/.ssh/id_rsa
sshkey=/var/lib/zuul/ssh/id_rsa
[zuul]
layout_config=/etc/zuul/layout.yaml

20
modules/zuul/templates/zuul.vhost.erb Normal file
View File

@ -0,0 +1,20 @@
<VirtualHost <%= scope.lookupvar("::zuul::vhost_name") %>:80>
ServerAdmin <%= scope.lookupvar("::zuul::serveradmin") %>
ErrorLog ${APACHE_LOG_DIR}/<%= scope.lookupvar("::zuul::vhost_name") %>-error.log
LogLevel warn
CustomLog ${APACHE_LOG_DIR}/<%= scope.lookupvar("::zuul::vhost_name") %>-access.log combined
RewriteEngine on
RewriteRule /zuul/status http://127.0.0.1:8001/status [P]
RewriteRule / http://127.0.0.1:8001/status [P]
SetEnv GIT_PROJECT_ROOT /var/lib/zuul/git/
SetEnv GIT_HTTP_EXPORT_ALL
AliasMatch ^/p/(.*/objects/[0-9a-f]{2}/[0-9a-f]{38})$ /var/lib/zuul/git/$1
AliasMatch ^/p/(.*/objects/pack/pack-[0-9a-f]{40}.(pack|idx))$ /var/lib/zuul/git/$1
ScriptAlias /p/ /usr/lib/git-core/git-http-backend/
</VirtualHost>