Add Zuul merge servers

Remove obsolete config options from zuul module.

The server and merger classes are constructed so that they
may coexist on a single server.  Also, the init section is
constructed so that it will install everything needed for
both services but without activating them (leaving maximum
flexibility for the operator).

Change-Id: I7b86fbbe4611c5edfb463a0a6944e0717f664188

manifests/site.pp

@@ -518,10 +518,30 @@ node 'zuul.openstack.org' {
       'jenkins06.openstack.org',
       'jenkins07.openstack.org',
       'jenkins-dev.openstack.org',
+      'zm01.openstack.org',
+      'zm02.openstack.org',
     ],
   }
 }
 
+node 'zm01.openstack.org' {
+  class { 'openstack_project::zuul_merger':
+    gerrit_server        => 'review.openstack.org',
+    gerrit_user          => 'jenkins',
+    zuul_ssh_private_key => hiera('jenkins_ssh_private_key_contents'),
+    sysadmins            => hiera('sysadmins'),
+  }
+}
+
+node 'zm02.openstack.org' {
+  class { 'openstack_project::zuul_merger':
+    gerrit_server        => 'review.openstack.org',
+    gerrit_user          => 'jenkins',
+    zuul_ssh_private_key => hiera('jenkins_ssh_private_key_contents'),
+    sysadmins            => hiera('sysadmins'),
+  }
+}
+
 node 'zuul-dev.openstack.org' {
   class { 'openstack_project::zuul_dev':
     gerrit_server        => 'review-dev.openstack.org',

modules/openstack_project/manifests/zuul_dev.pp

@@ -28,12 +28,14 @@ class openstack_project::zuul_dev(
     zuul_ssh_private_key => $zuul_ssh_private_key,
     url_pattern          => $url_pattern,
     zuul_url             => $zuul_url,
-    push_change_refs     => false,
     job_name_in_report   => true,
     status_url           => 'http://zuul-dev.openstack.org/',
     statsd_host          => $statsd_host,
   }
 
+  class { '::zuul::server': }
+  class { '::zuul::merger': }
+
   file { '/etc/zuul/layout.yaml':
     ensure => present,
     source => 'puppet:///modules/openstack_project/zuul/layout-dev.yaml',
@@ -58,6 +60,11 @@ class openstack_project::zuul_dev(
     notify => Exec['zuul-reload'],
   }
 
+  file { '/etc/zuul/merger-logging.conf':
+    ensure => present,
+    source => 'puppet:///modules/openstack_project/zuul/merger-logging.conf',
+  }
+
   class { '::recheckwatch':
     gerrit_server                => $gerrit_server,
     gerrit_user                  => $gerrit_user,

modules/openstack_project/manifests/zuul_merger.pp

@@ -0,0 +1,31 @@
+# == Class: openstack_project::zuul_merger
+#
+class openstack_project::zuul_merger(
+  $vhost_name = $::fqdn,
+  $gerrit_server = '',
+  $gerrit_user = '',
+  $zuul_ssh_private_key = '',
+  $zuul_url = "http://${::fqdn}/p",
+  $sysadmins = [],
+) {
+
+  class { 'openstack_project::server':
+    iptables_public_tcp_ports => [80],
+    sysadmins                 => $sysadmins,
+  }
+
+  class { '::zuul':
+    vhost_name           => $vhost_name,
+    gerrit_server        => $gerrit_server,
+    gerrit_user          => $gerrit_user,
+    zuul_ssh_private_key => $zuul_ssh_private_key,
+    zuul_url             => $zuul_url,
+  }
+
+  class { '::zuul::merger': }
+
+  file { '/etc/zuul/merger-logging.conf':
+    ensure => present,
+    source => 'puppet:///modules/openstack_project/zuul/merger-logging.conf',
+  }
+}

modules/openstack_project/manifests/zuul_prod.pp

@@ -10,7 +10,6 @@ class openstack_project::zuul_prod(
   $sysadmins = [],
   $statsd_host = '',
   $gearman_workers = [],
-  $replication_targets = []
 ) {
   # Turn a list of hostnames into a list of iptables rules
   $iptables_rules = regsubst ($gearman_workers, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 4730 -s \1 -j ACCEPT')
@@ -29,13 +28,13 @@ class openstack_project::zuul_prod(
     zuul_ssh_private_key => $zuul_ssh_private_key,
     url_pattern          => $url_pattern,
     zuul_url             => $zuul_url,
-    push_change_refs     => false,
     job_name_in_report   => true,
     status_url           => 'http://status.openstack.org/zuul/',
     statsd_host          => $statsd_host,
-    replication_targets  => $replication_targets,
   }
 
+  class { '::zuul::server': }
+
   file { '/etc/zuul/layout.yaml':
     ensure => present,
     source => 'puppet:///modules/openstack_project/zuul/layout.yaml',
@@ -60,6 +59,11 @@ class openstack_project::zuul_prod(
     notify => Exec['zuul-reload'],
   }
 
+  file { '/etc/zuul/merger-logging.conf':
+    ensure => present,
+    source => 'puppet:///modules/openstack_project/zuul/merger-logging.conf',
+  }
+
   class { '::recheckwatch':
     gerrit_server                => $gerrit_server,
     gerrit_user                  => $gerrit_user,

modules/zuul/files/zuul-merger.init

@@ -0,0 +1,134 @@
+#! /bin/sh
+### BEGIN INIT INFO
+# Provides:          zuul
+# Required-Start:    $remote_fs $syslog
+# Required-Stop:     $remote_fs $syslog
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Zuul
+# Description:       Trunk gating system
+### END INIT INFO
+
+# Do NOT "set -e"
+
+# PATH should only include /usr/* if it runs after the mountnfs.sh script
+PATH=/sbin:/usr/sbin:/bin:/usr/bin
+DESC="Zuul"
+NAME=zuul-merger
+DAEMON=/usr/local/bin/zuul-merger
+PIDFILE=/var/run/$NAME/$NAME.pid
+SCRIPTNAME=/etc/init.d/$NAME
+USER=zuul
+
+# Exit if the package is not installed
+[ -x "$DAEMON" ] || exit 0
+
+# Read configuration variable file if it is present
+[ -r /etc/default/$NAME ] && . /etc/default/$NAME
+
+# Load the VERBOSE setting and other rcS variables
+. /lib/init/vars.sh
+
+# Define LSB log_* functions.
+# Depend on lsb-base (>= 3.0-6) to ensure that this file is present.
+. /lib/lsb/init-functions
+
+#
+# Function that starts the daemon/service
+#
+do_start()
+{
+	# Return
+	#   0 if daemon has been started
+	#   1 if daemon was already running
+	#   2 if daemon could not be started
+
+        mkdir -p /var/run/$NAME
+	chown $USER /var/run/$NAME
+	ulimit -n 8192
+	start-stop-daemon --start --quiet --pidfile $PIDFILE -c $USER --exec $DAEMON --test > /dev/null \
+		|| return 1
+	start-stop-daemon --start --quiet --pidfile $PIDFILE -c $USER --exec $DAEMON -- \
+		$DAEMON_ARGS \
+		|| return 2
+	# Add code here, if necessary, that waits for the process to be ready
+	# to handle requests from services started subsequently which depend
+	# on this one.  As a last resort, sleep for some time.
+}
+
+#
+# Function that stops the daemon/service
+#
+do_stop()
+{
+	# Return
+	#   0 if daemon has been stopped
+	#   1 if daemon was already stopped
+	#   2 if daemon could not be stopped
+	#   other if a failure occurred
+	start-stop-daemon --stop --signal 9 --pidfile $PIDFILE
+	RETVAL="$?"
+	[ "$RETVAL" = 2 ] && return 2
+	rm -f /var/run/$NAME/*
+	return "$RETVAL"
+}
+ 
+#
+# Function that sends a SIGHUP to the daemon/service
+#
+do_reload() {
+	#
+	# If the daemon can reload its configuration without
+	# restarting (for example, when it is sent a SIGHUP),
+	# then implement that here.
+	#
+	start-stop-daemon --stop --signal 1 --quiet --pidfile $PIDFILE --name zuul-merger
+	return 0
+}
+
+case "$1" in
+  start)
+	[ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME"
+	do_start
+	case "$?" in
+		0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;;
+		2) [ "$VERBOSE" != no ] && log_end_msg 1 ;;
+	esac
+	;;
+  stop)
+	[ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME"
+	do_stop
+	case "$?" in
+		0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;;
+		2) [ "$VERBOSE" != no ] && log_end_msg 1 ;;
+	esac
+	;;
+  status)
+       status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $?
+       ;;
+  reload|force-reload)
+	#
+	# If do_reload() is not implemented then leave this commented out
+	# and leave 'force-reload' as an alias for 'restart'.
+	#
+	log_daemon_msg "Reloading $DESC" "$NAME"
+	do_reload
+	log_end_msg $?
+	;;
+  restart)
+	#
+	# If the "reload" option is implemented then remove the
+	# 'force-reload' alias
+	#
+	log_daemon_msg "Restarting $DESC" "$NAME"
+	do_stop
+	do_start
+	;;
+  *)
+	#echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2
+	echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2
+	exit 3
+	;;
+esac
+
+:

modules/zuul/manifests/init.pp

@@ -29,11 +29,9 @@ class zuul (
   $status_url = "https://${::fqdn}/",
   $zuul_url = '',
   $git_source_repo = 'https://git.openstack.org/openstack-infra/zuul',
-  $push_change_refs = false,
   $job_name_in_report = false,
   $revision = 'master',
   $statsd_host = '',
-  $replication_targets = []
 ) {
   include apache
   include pip
@@ -132,6 +130,13 @@ class zuul (
     require => User['zuul'],
   }
 
+  file { '/var/run/zuul-merger':
+    ensure  => directory,
+    owner   => 'zuul',
+    group   => 'zuul',
+    require => User['zuul'],
+  }
+
   file { '/var/lib/zuul':
     ensure  => directory,
     owner   => 'zuul',
@@ -210,27 +215,12 @@ class zuul (
     source => 'puppet:///modules/zuul/zuul.init',
   }
 
-  exec { 'zuul-reload':
+  file { '/etc/init.d/zuul-merger':
-    command     => '/etc/init.d/zuul reload',
+    ensure => present,
-    require     => File['/etc/init.d/zuul'],
+    owner  => 'root',
-    refreshonly => true,
+    group  => 'root',
-  }
+    mode   => '0555',
-
+    source => 'puppet:///modules/zuul/zuul-merger.init',
-  service { 'zuul':
-    name       => 'zuul',
-    enable     => true,
-    hasrestart => true,
-    require    => File['/etc/init.d/zuul'],
-  }
-
-  cron { 'zuul_repack':
-    user        => 'zuul',
-    hour        => '4',
-    minute      => '7',
-    command     => 'find /var/lib/zuul/git/ -maxdepth 3 -type d -name ".git" -exec git --git-dir="{}" pack-refs --all \;',
-    environment => 'PATH=/usr/bin:/bin:/usr/sbin:/sbin',
-    require     => [User['zuul'],
-                    File['/var/lib/zuul/git']],
   }
 
   apache::vhost { $vhost_name:
@@ -248,5 +238,4 @@ class zuul (
   a2mod { 'proxy_http':
     ensure => present,
   }
-
 }

modules/zuul/manifests/merger.pp

@@ -0,0 +1,36 @@
+# Copyright 2012-2013 Hewlett-Packard Development Company, L.P.
+# Copyright 2014 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# == Class: zuul::merger
+#
+class zuul::merger (
+) {
+  service { 'zuul-merger':
+    name       => 'zuul-merger',
+    enable     => true,
+    hasrestart => true,
+    require    => File['/etc/init.d/zuul-merger'],
+  }
+
+  cron { 'zuul_repack':
+    user        => 'zuul',
+    hour        => '4',
+    minute      => '7',
+    command     => 'find /var/lib/zuul/git/ -maxdepth 3 -type d -name ".git" -exec git --git-dir="{}" pack-refs --all \;',
+    environment => 'PATH=/usr/bin:/bin:/usr/sbin:/sbin',
+    require     => [User['zuul'],
+                    File['/var/lib/zuul/git']],
+  }
+}

modules/zuul/manifests/server.pp

@@ -0,0 +1,32 @@
+# Copyright 2012-2013 Hewlett-Packard Development Company, L.P.
+# Copyright 2014 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# == Class: zuul::server
+#
+class zuul::server (
+) {
+  service { 'zuul':
+    name       => 'zuul',
+    enable     => true,
+    hasrestart => true,
+    require    => File['/etc/init.d/zuul'],
+  }
+
+  exec { 'zuul-reload':
+    command     => '/etc/init.d/zuul reload',
+    require     => File['/etc/init.d/zuul'],
+    refreshonly => true,
+  }
+}

modules/zuul/templates/zuul.conf.erb

@@ -15,15 +15,12 @@ layout_config=/etc/zuul/layout.yaml
 log_config=/etc/zuul/logging.conf
 state_dir=/var/lib/zuul
 git_dir=/var/lib/zuul/git
-push_change_refs=<%= push_change_refs %>
 url_pattern=<%= url_pattern %>
 status_url=<%= status_url %>
 job_name_in_report=<%= job_name_in_report %>
 zuul_url=<%= zuul_url %>
 
-<% if replication_targets != [] -%>
+[merger]
-[replication]
+git_dir=/var/lib/zuul/git
-<% replication_targets.each do |target| -%>
+zuul_url=<%= zuul_url %>
-<%= target['name'] %>=<%= target['url'] %>
+log_config=/etc/zuul/merger-logging.conf
-<% end -%>
-<% end -%>