Initial support for zuulv2.5

Change-Id: Ic0629f3a7c957905c6241d4e7898a4f12e3ece75
Signed-off-by: Paul Belanger <pabelanger@redhat.com>

defaults/main.yaml

@@ -29,6 +29,12 @@ zuul_file_gearman_logging_conf_mode: "0644"
 zuul_file_gearman_logging_conf_owner: "{{ zuul_user_name }}"
 zuul_file_gearman_logging_conf_src: etc/zuul/gearman-logging.conf
 
+zuul_file_launcher_logging_conf_dest: /etc/zuul/launcher-logging.conf
+zuul_file_launcher_logging_conf_group: "{{ zuul_user_group }}"
+zuul_file_launcher_logging_conf_mode: "0644"
+zuul_file_launcher_logging_conf_owner: "{{ zuul_user_name }}"
+zuul_file_launcher_logging_conf_src: etc/zuul/launcher-logging.conf
+
 zuul_file_merger_logging_conf_dest: /etc/zuul/merger-logging.conf
 zuul_file_merger_logging_conf_group: "{{ zuul_user_group }}"
 zuul_file_merger_logging_conf_mode: "0644"
@@ -60,12 +66,20 @@ zuul_pip_version:
 zuul_file_zuul_service_config_group: root
 zuul_file_zuul_service_config_owner: root
 
+zuul_file_zuul_launcher_service_group: root
+zuul_file_zuul_launcher_service_owner: root
+
 zuul_file_zuul_merger_service_group: root
 zuul_file_zuul_merger_service_owner: root
 
 zuul_file_zuul_server_service_group: root
 zuul_file_zuul_server_service_owner: root
 
+zuul_service_zuul_launcher_enabled: yes
+zuul_service_zuul_launcher_manage: true
+zuul_service_zuul_launcher_name: zuul-launcher
+zuul_service_zuul_launcher_state: started
+
 zuul_service_zuul_merger_enabled: yes
 zuul_service_zuul_merger_manage: true
 zuul_service_zuul_merger_name: zuul-merger

files/etc/init.d/zuul-launcher

@@ -0,0 +1,140 @@
+#! /bin/sh
+### BEGIN INIT INFO
+# Provides:          zuul
+# Required-Start:    $remote_fs $syslog
+# Required-Stop:     $remote_fs $syslog
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Zuul
+# Description:       Trunk gating system launcher
+### END INIT INFO
+
+# Do NOT "set -e"
+
+# PATH should only include /usr/* if it runs after the mountnfs.sh script
+PATH=/sbin:/usr/sbin:/bin:/usr/bin
+DESC="Zuul Launcher"
+NAME=zuul-launcher
+DAEMON=/usr/local/bin/zuul-launcher
+PIDFILE=/var/run/$NAME/$NAME.pid
+SCRIPTNAME=/etc/init.d/$NAME
+USER=zuul
+
+# Exit if the package is not installed
+[ -x "$DAEMON" ] || exit 0
+
+# Read configuration variable file if it is present
+[ -r /etc/default/$NAME ] && . /etc/default/$NAME
+
+# Load the VERBOSE setting and other rcS variables
+. /lib/init/vars.sh
+
+# Define LSB log_* functions.
+# Depend on lsb-base (>= 3.0-6) to ensure that this file is present.
+. /lib/lsb/init-functions
+
+#
+# Function that starts the daemon/service
+#
+do_start()
+{
+    # Return
+    #   0 if daemon has been started
+    #   1 if daemon was already running
+    #   2 if daemon could not be started
+    #   3 if pid file already exist
+
+    mkdir -p /var/run/$NAME
+    chown $USER /var/run/$NAME
+    if [ -f $PIDFILE ]; then
+        return 3
+    fi
+    start-stop-daemon \
+        --start --quiet --pidfile $PIDFILE -c $USER \
+        --exec $DAEMON --test > /dev/null || return 1
+    start-stop-daemon \
+        --start --quiet --pidfile $PIDFILE -c $USER \
+        --exec $DAEMON -- $DAEMON_ARGS || return 2
+    # Add code here, if necessary, that waits for the process to be ready
+    # to handle requests from services started subsequently which depend
+    # on this one.  As a last resort, sleep for some time.
+}
+
+#
+# Function that stops the daemon/service
+#
+do_stop()
+{
+    # Return
+    #   0 if daemon has been stopped
+    #   1 if daemon was already stopped
+    #   2 if daemon could not be stopped
+    #   other if a failure occurred
+    start-stop-daemon --stop --signal 9 --pidfile $PIDFILE
+    RETVAL="$?"
+    [ "$RETVAL" = 2 ] && return 2
+    rm -f /var/run/$NAME/*
+    return "$RETVAL"
+}
+
+#
+# Function that sends a SIGHUP to the daemon/service
+#
+do_reload() {
+    #
+    # If the daemon can reload its configuration without
+    # restarting (for example, when it is sent a SIGHUP),
+    # then implement that here.
+    #
+    start-stop-daemon \
+        --stop --signal 1 --quiet --pidfile $PIDFILE --name $DAEMON
+    return 0
+}
+
+case "$1" in
+    start)
+        [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME"
+        do_start
+        case "$?" in
+            0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;;
+            2) [ "$VERBOSE" != no ] && log_end_msg 1 ;;
+            3) echo "Pidfile at $PIDFILE already exists, run service zuul-launcher stop to clean up."
+        esac
+        ;;
+    stop)
+        [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME"
+        do_stop
+        case "$?" in
+            0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;;
+            2) [ "$VERBOSE" != no ] && log_end_msg 1 ;;
+        esac
+        ;;
+    status)
+       status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $?
+       ;;
+    reload|force-reload)
+        #
+        # If do_reload() is not implemented then leave this commented out
+        # and leave 'force-reload' as an alias for 'restart'.
+        #
+        log_daemon_msg "Reloading $DESC" "$NAME"
+        do_reload
+        log_end_msg $?
+        ;;
+    restart)
+        #
+        # If the "reload" option is implemented then remove the
+        # 'force-reload' alias
+        #
+        log_daemon_msg "Restarting $DESC" "$NAME"
+        do_stop
+        do_start
+        ;;
+    *)
+        #echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2
+        echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2
+        exit 3
+        ;;
+esac
+
+:

files/etc/systemd/system/zuul-launcher.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Zuul Launcher Service
+After=syslog.target network.target
+
+[Service]
+Type=simple
+User=zuul
+Group=zuul
+EnvironmentFile=-/etc/sysconfig/zuul
+ExecStart=/usr/bin/zuul-launcher -d
+
+[Install]
+WantedBy=multi-user.target

files/etc/zuul/launcher-logging.conf

@@ -0,0 +1,49 @@
+[loggers]
+keys=root,zuul,gerrit,gear
+
+[handlers]
+keys=console,debug,normal
+
+[formatters]
+keys=simple
+
+[logger_root]
+level=WARNING
+handlers=console
+
+[logger_zuul]
+level=DEBUG
+handlers=debug,normal
+qualname=zuul
+
+[logger_gerrit]
+level=INFO
+handlers=debug,normal
+qualname=gerrit
+
+[logger_gear]
+level=WARNING
+handlers=debug,normal
+qualname=gear
+
+[handler_console]
+level=WARNING
+class=StreamHandler
+formatter=simple
+args=(sys.stdout,)
+
+[handler_debug]
+level=DEBUG
+class=logging.handlers.WatchedFileHandler
+formatter=simple
+args=('/var/log/zuul/launcher-debug.log',)
+
+[handler_normal]
+level=INFO
+class=logging.handlers.WatchedFileHandler
+formatter=simple
+args=('/var/log/zuul/launcher.log',)
+
+[formatter_simple]
+format=%(asctime)s %(levelname)s %(name)s: %(message)s
+datefmt=

files/etc/zuul/zuul.conf

@@ -22,6 +22,10 @@ log_config = /etc/zuul/server-logging.conf
 pidfile = /var/run/zuul-server/zuul-server.pid
 state_dir = /var/lib/zuul
 
+[launcher]
+jenkins_jobs=/var/lib/zuul
+workspace_root=/var/lib/zuul/workspace
+
 [merger]
 git_dir = /var/lib/zuul/git
 log_config = /etc/zuul/merger-logging.conf

handlers/main.yaml

@@ -12,6 +12,15 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 ---
+- name: Reload zuul-launcher
+  become: yes
+  service:
+    name: zuul-launcher
+    state: reloaded
+  when: zuul_service_zuul_launcher_manage and
+        zuul_service_zuul_launcher_state == "started" and not
+        __zuul_launcher_service.changed
+
 - name: Reload zuul-merger
   become: yes
   service:

tasks/config.yaml

@@ -33,6 +33,16 @@
     src: "{{ zuul_file_gearman_logging_conf_src }}"
   notify: Reload zuul-server
 
+- name: Copy launcher logging file.
+  become: yes
+  copy:
+    dest: "{{ zuul_file_launcher_logging_conf_dest }}"
+    group: "{{ zuul_file_launcher_logging_conf_group }}"
+    mode: "{{ zuul_file_launcher_logging_conf_mode }}"
+    owner: "{{ zuul_file_launcher_logging_conf_owner }}"
+    src: "{{ zuul_file_launcher_logging_conf_src }}"
+  notify: Reload zuul-launcher
+
 - name: Copy merger logging file.
   become: yes
   copy:
@@ -72,5 +82,6 @@
     owner: "{{ zuul_file_zuul_conf_owner }}"
     src: "{{ zuul_file_zuul_conf_src }}"
   notify:
+    - Reload zuul-launcher
     - Reload zuul-merger
     - Reload zuul-server

tasks/install.yaml

@@ -33,3 +33,13 @@
   when: zuul_install_method == 'git' or zuul_install_method == 'pip'
 
 - include: "install/{{ zuul_install_method }}.yaml"
+
+# TODO(pabelanger): Remove once https://review.openstack.org/#/c/316387/ has
+# been merged.
+- name: Install missing pip dependencies
+  pip:
+    name: "{{ item }}"
+  with_items:
+    - jenkins-job-builder
+    - pyzmq
+  when: zuul_install_method == 'git' or zuul_install_method == 'pip'

tasks/service.yaml

@@ -27,6 +27,21 @@
     zuul_file_zuul_service_config_src: "{{ __zuul_file_zuul_service_config_src }}"
   when: zuul_file_zuul_service_config_src is not defined
 
+- name: Define zuul_file_zuul_launcher_service_dest.
+  set_fact:
+    zuul_file_zuul_launcher_service_dest: "{{ __zuul_file_zuul_launcher_service_dest }}"
+  when: zuul_file_zuul_launcher_service_dest is not defined
+
+- name: Define zuul_file_zuul_launcher_service_mode.
+  set_fact:
+    zuul_file_zuul_launcher_service_mode: "{{ __zuul_file_zuul_launcher_service_mode }}"
+  when: zuul_file_zuul_launcher_service_mode is not defined
+
+- name: Define zuul_file_zuul_launcher_service_src.
+  set_fact:
+    zuul_file_zuul_launcher_service_src: "{{ __zuul_file_zuul_launcher_service_src }}"
+  when: zuul_file_zuul_launcher_service_src is not defined
+
 - name: Define zuul_file_zuul_merger_service_dest.
   set_fact:
     zuul_file_zuul_merger_service_dest: "{{ __zuul_file_zuul_merger_service_dest }}"
@@ -65,6 +80,14 @@
     owner: "{{ zuul_file_zuul_service_config_owner }}"
     src: "{{ zuul_file_zuul_service_config_src }}"
 
+- name: Copy zuul-launcher service into place.
+  copy:
+    dest: "{{ zuul_file_zuul_launcher_service_dest }}"
+    group: "{{ zuul_file_zuul_launcher_service_group }}"
+    mode: "{{ zuul_file_zuul_launcher_service_mode }}"
+    owner: "{{ zuul_file_zuul_launcher_service_owner }}"
+    src: "{{ zuul_file_zuul_launcher_service_src }}"
+
 - name: Copy zuul-merger service into place.
   copy:
     dest: "{{ zuul_file_zuul_merger_service_dest }}"
@@ -81,6 +104,14 @@
     owner: "{{ zuul_file_zuul_server_service_owner }}"
     src: "{{ zuul_file_zuul_server_service_src }}"
 
+- name: Enable zuul-launcher service.
+  service:
+    enabled: "{{ zuul_service_zuul_launcher_enabled }}"
+    name: "{{ zuul_service_zuul_launcher_name }}"
+    state: "{{ zuul_service_zuul_launcher_state }}"
+  register: __zuul_launcher_service
+  when: zuul_service_zuul_launcher_manage
+
 - name: Enable zuul-merger service.
   service:
     enabled: "{{ zuul_service_zuul_merger_enabled }}"

tests/test.yaml

@@ -120,6 +120,22 @@
           - debian_zuul_service_config_stat.stat.mode == '0644'
       when: ansible_os_family == 'Debian'
 
+    - name: Register /etc/init.d/zuul-launcher
+      stat:
+        path: /etc/init.d/zuul-launcher
+      register: debian_zuul_launcher_service_stat
+      when: ansible_os_family == 'Debian'
+
+    - name: Assert debian_zuul_launcher_service_stat tests.
+      assert:
+        that:
+          - debian_zuul_launcher_service_stat.stat.exists
+          - debian_zuul_launcher_service_stat.stat.isreg
+          - debian_zuul_launcher_service_stat.stat.pw_name == 'root'
+          - debian_zuul_launcher_service_stat.stat.gr_name == 'root'
+          - debian_zuul_launcher_service_stat.stat.mode == '0755'
+      when: ansible_os_family == 'Debian'
+
     - name: Register /etc/init.d/zuul-merger
       stat:
         path: /etc/init.d/zuul-merger
@@ -168,6 +184,22 @@
           - redhat_zuul_service_config_stat.stat.mode == '0644'
       when: ansible_os_family == 'RedHat'
 
+    - name: Register /etc/systemd/system/zuul-launcher.service
+      stat:
+        path: /etc/systemd/system/zuul-launcher.service
+      register: redhat_zuul_launcher_service_stat
+      when: ansible_os_family == 'RedHat'
+
+    - name: Assert redhat_zuul_launcher_service_stat tests.
+      assert:
+        that:
+          - redhat_zuul_launcher_service_stat.stat.exists
+          - redhat_zuul_launcher_service_stat.stat.isreg
+          - redhat_zuul_launcher_service_stat.stat.pw_name == 'root'
+          - redhat_zuul_launcher_service_stat.stat.gr_name == 'root'
+          - redhat_zuul_launcher_service_stat.stat.mode == '0644'
+      when: ansible_os_family == 'RedHat'
+
     - name: Register /etc/systemd/system/zuul-merger.service
       stat:
         path: /etc/systemd/system/zuul-merger.service
@@ -200,6 +232,10 @@
           - redhat_zuul_server_service_stat.stat.mode == '0644'
       when: ansible_os_family == 'RedHat'
 
+    - name: Ensure zuul-launcher is running.
+      shell: /usr/sbin/service zuul-launcher status
+      tags: skip_ansible_lint
+
     - name: Ensure zuul-merger is running.
       shell: /usr/sbin/service zuul-merger status
       tags: skip_ansible_lint

vars/Debian.yaml

@@ -21,6 +21,10 @@ __zuul_file_zuul_service_config_dest: /etc/default/zuul
 __zuul_file_zuul_service_config_mode: "0644"
 __zuul_file_zuul_service_config_src: etc/default/zuul
 
+__zuul_file_zuul_launcher_service_dest: /etc/init.d/zuul-launcher
+__zuul_file_zuul_launcher_service_mode: "0755"
+__zuul_file_zuul_launcher_service_src: etc/init.d/zuul-launcher
+
 __zuul_file_zuul_merger_service_dest: /etc/init.d/zuul-merger
 __zuul_file_zuul_merger_service_mode: "0755"
 __zuul_file_zuul_merger_service_src: etc/init.d/zuul-merger

vars/RedHat.yaml

@@ -23,6 +23,10 @@ __zuul_file_zuul_service_config_dest: /etc/sysconfig/zuul
 __zuul_file_zuul_service_config_mode: "0644"
 __zuul_file_zuul_service_config_src: etc/sysconfig/zuul
 
+__zuul_file_zuul_launcher_service_dest: /etc/systemd/system/zuul-launcher.service
+__zuul_file_zuul_launcher_service_mode: "0644"
+__zuul_file_zuul_launcher_service_src: etc/systemd/system/zuul-launcher.service
+
 __zuul_file_zuul_merger_service_dest: /etc/systemd/system/zuul-merger.service
 __zuul_file_zuul_merger_service_mode: "0644"
 __zuul_file_zuul_merger_service_src: etc/systemd/system/zuul-merger.service