Run Zuul using Ansible and Containers

Zuul is publishing lovely container images, so we should go ahead and start using them. We can't use containers for zuul-executor because of the docker->bubblewrap->AFS issue, so install from pip there. Don't start any of the containers by default, which should let us safely roll this out and then do a rolling restart. For things (like web or mergers) where it's safe to do so, a followup change will swap the flag. Change-Id: I37dcce3a67477ad3b2c36f2fd3657af18bc25c40
2020-04-05 09:25:28 -05:00 · 2020-04-05 09:25:28 -05:00 · f0b77485ec
commit f0b77485ec
parent b0ab2f37c5
55 changed files with 1698 additions and 610 deletions
--- a/.zuul.yaml
+++ b/.zuul.yaml
@ -1397,6 +1397,39 @@
      - playbooks/roles/zuul-preview/
      - testinfra/test_zuul_preview.py

+- job:
+    name: system-config-run-zuul
+    parent: system-config-run
+    description: |
+      Run the playbook for the docker registry.
+    nodeset:
+      nodes:
+        - name: bridge.openstack.org
+          label: ubuntu-bionic
+        - name: zk01.opendev.org
+          label: ubuntu-bionic
+        - name: zm01.openstack.org
+          label: ubuntu-xenial
+        - name: zl01.openstack.org
+          label: ubuntu-xenial
+        - name: zuul01.openstack.org
+          label: ubuntu-xenial
+    vars:
+      run_playbooks:
+        - playbooks/service-letsencrypt.yaml
+        - playbooks/service-zookeeper.yaml
+        - playbooks/service-zuul.yaml
+    files:
+      - playbooks/install-ansible.yaml
+      - playbooks/service-zookeeper.yaml
+      - playbooks/service-zuul.yaml
+      - playbooks/group_vars/zuul
+      - playbooks/group_vars/zookeeper.yaml
+      - playbooks/host_vars/zk\d+
+      - playbooks/host_vars/zuul01.openstack.org
+      - playbooks/roles/zookeeper/
+      - playbooks/roles/zuul
+
 - job:
    name: system-config-run-review
    parent: system-config-run-containers
@ -2165,6 +2198,7 @@
              - name: system-config-build-image-gerrit-2.13
                soft: true
        - system-config-run-zookeeper
+        - system-config-run-zuul
        - system-config-run-zuul-preview
        - system-config-run-letsencrypt
        - system-config-build-image-jinja-init:
@ -2230,6 +2264,7 @@
              - name: system-config-upload-image-gerrit-2.13
                soft: true
        - system-config-run-zookeeper
+        - system-config-run-zuul
        - system-config-run-zuul-preview
        - system-config-run-letsencrypt
        - system-config-upload-image-jinja-init:
--- a/hiera/common.yaml
+++ b/hiera/common.yaml
@ -433,56 +433,3 @@ mosquitto_tls_ca_file: |
  c4g/VhsxOBi0cQ+azcgOno4uG+GMmIPLHzHxREzGBHNJdmAPx/i9F4BrLunMTA5a
  mnkPIAou1Z5jJh5VkpTYghdae9C8x49OhgQ=
  -----END CERTIFICATE-----
-gearman_client_ssl_cert: |
-  -----BEGIN CERTIFICATE-----
-  MIIEYTCCA0mgAwIBAgIJAKkAn3gh0LBQMA0GCSqGSIb3DQEBCwUAMIG5MQswCQYD
-  VQQGEwJVUzEOMAwGA1UECAwFVGV4YXMxDzANBgNVBAcMBkF1c3RpbjEdMBsGA1UE
-  CgwUT3BlblN0YWNrIEZvdW5kYXRpb24xFzAVBgNVBAsMDkluZnJhc3RydWN0dXJl
-  MR0wGwYDVQQDDBR6dXVsdjMub3BlbnN0YWNrLm9yZzEyMDAGCSqGSIb3DQEJARYj
-  b3BlbnN0YWNrLWluZnJhQGxpc3RzLm9wZW5zdGFjay5vcmcwHhcNMTcwNjE2MjMw
-  MjQyWhcNMjcwNjE0MjMwMjQyWjCBszELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBVRl
-  eGFzMQ8wDQYDVQQHDAZBdXN0aW4xHTAbBgNVBAoMFE9wZW5TdGFjayBGb3VuZGF0
-  aW9uMRcwFQYDVQQLDA5JbmZyYXN0cnVjdHVyZTEXMBUGA1UEAwwOZ2Vhcm1hbi5j
-  bGllbnQxMjAwBgkqhkiG9w0BCQEWI29wZW5zdGFjay1pbmZyYUBsaXN0cy5vcGVu
-  c3RhY2sub3JnMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsh3qSWIp
-  w6kXS4IIPU7fPP2felHCtmZyfgKolYbq1iVafcc/EUHa1onlaM+w7OEHr68y3Qau
-  SY6ifEsUWCKJlhu+UlHGwVIZliL02+9EAZ1DDs6OtxKa7nOIkWq8P8kRex234QVd
-  y37+vV+/lDeCbLoGo5P0j51fnqy10afg2xRblmXgqeqaiJAvCmEnG9S9q9+gbisZ
-  1D2r+JtoTUMZtPY9NomvgdNuwmF5+VeO+CQepRWlA+0ysCFVgVwm++PNXETadHOj
-  mOSJxiq2u6fysZb7ctHgGuu+Ce3PVwah+kK/PEXADs7SjhJruSmL1ap2izc6kTFW
-  GSU/wkkPXtbWJwIDAQABo3AwbjAJBgNVHRMEAjAAMCEGCWCGSAGG+EIBDQQUFhJj
-  bGllbnQgY2VydGlmaWNhdGUwHQYDVR0OBBYEFKTyA6hjUY8jNxOEM5zuU7qecogX
-  MB8GA1UdIwQYMBaAFFP8JfdXPn8mhZLaXMa8NQIJlmneMA0GCSqGSIb3DQEBCwUA
-  A4IBAQAiLYckNAx7GQGCSXC92R23o181FiCePuNAgCb4QsaQkA/JopaLrn11R33Y
-  XO1C5fvsopKvcmEJKX0BJwNy41tz/rNmKXYy4hsPKYMsNgJQtYe98Mp+VHgAmtZ3
-  U0v49mUJA4YiLs/QmB6bmLknl1XjzJvbLu3gfVSGsquDXN1TcHLZy2fQlD6/D7HF
-  2Zj44Af4b2xFcZc7J/iErIj8LGHx3alkGAgdXw+SQkgzDeXC/DhrXC1jVJQQQzfU
-  /4GjbLiPBLb+QIAaBVv+iVVok22DSvMydjI4Zr89NXDWEOZc8oZ7nBf9Sv1+I0xB
-  6YQoN+t1YSx3G8AxPSZwyGlwhZo0
-  -----END CERTIFICATE-----
-gearman_ssl_ca: |
-  -----BEGIN CERTIFICATE-----
-  MIIERzCCAy+gAwIBAgIJAKkAn3gh0LBOMA0GCSqGSIb3DQEBCwUAMIG5MQswCQYD
-  VQQGEwJVUzEOMAwGA1UECAwFVGV4YXMxDzANBgNVBAcMBkF1c3RpbjEdMBsGA1UE
-  CgwUT3BlblN0YWNrIEZvdW5kYXRpb24xFzAVBgNVBAsMDkluZnJhc3RydWN0dXJl
-  MR0wGwYDVQQDDBR6dXVsdjMub3BlbnN0YWNrLm9yZzEyMDAGCSqGSIb3DQEJARYj
-  b3BlbnN0YWNrLWluZnJhQGxpc3RzLm9wZW5zdGFjay5vcmcwHhcNMTcwNjE2MjA1
-  MjA3WhcNMjAwNjE1MjA1MjA3WjCBuTELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBVRl
-  eGFzMQ8wDQYDVQQHDAZBdXN0aW4xHTAbBgNVBAoMFE9wZW5TdGFjayBGb3VuZGF0
-  aW9uMRcwFQYDVQQLDA5JbmZyYXN0cnVjdHVyZTEdMBsGA1UEAwwUenV1bHYzLm9w
-  ZW5zdGFjay5vcmcxMjAwBgkqhkiG9w0BCQEWI29wZW5zdGFjay1pbmZyYUBsaXN0
-  cy5vcGVuc3RhY2sub3JnMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA
-  zTnzmZkB/P+C0eHFmPyU8myEmubRVw2vK1aqx0Y7bFMlXAVH6CodI6r4VpS4vGPL
-  AfBGAmIZJlBuRysZHW3J6GuzhBFyBILHJX9PZkeJyHa3NU4ILDPMXAD/oWQnqlp1
-  3kYJ3xS1QWhPvaohC+Io3LErXOMp32mhrEmm3BGfWiXbV9STcseeLX6BKPdqBzaT
-  d8RFkrvsEJTTjwIJLreyrphrtXu/VS9uEMWaHj4/94lLXn8fn3CuUfs48kPDTlaw
-  vFg2lIGpfOui4s9Vhrafy1nrz1KzKHjhhnF80irrIo3kOkWaKeBuTyy7+MSx7PTi
-  5RgSoKTKyMbMA6nbCj73KQIDAQABo1AwTjAdBgNVHQ4EFgQUU/wl91c+fyaFktpc
-  xrw1AgmWad4wHwYDVR0jBBgwFoAUU/wl91c+fyaFktpcxrw1AgmWad4wDAYDVR0T
-  BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAe/6S1DWRtXwzBgwTCW7FR3IrpZzP
-  4eN3TUbJy6tvff+iY6+96WV9vyH62NU8oEn5TUqy8r+EiOchbXJq8pvlPAcwdaeC
-  a9pjJku40oVai0pncqDnF/WOiXNkW71bRs/qQtIuVwKwVm9OyizjWsQtjm4Ycpju
-  92liz5Q/ZZu+7eIufQYRr7lthgmTLCjqeS4qxiY7Y03ZLZpvEL+KVskkjPzHvzTO
-  S1Rq0t3ssb4uH78rvXj1Q/C2gVucUBE86P9AckSZtANGlmiKBnO6Lc1xQbsFyfSn
-  Xbt2g9IiP3nTEapCx/M8/Zl5M+XwK7pbQWdtwGnvGPoeFNV1sVT4iO1dLg==
-  -----END CERTIFICATE-----
--- a/hiera/group/zuul-merger.yaml
+++ b/hiera/group/zuul-merger.yaml
@ -1,50 +0,0 @@
---
-# TODO(pabelanger): This can be deleted once we migration to zuulv3.
-zuul_sites:
-  - name: 'tarballs.openstack.org'
-    host: 'tarballs.openstack.org'
-    user: 'jenkins'
-    root: '/srv/static'
-
-  - name: 'yaml2ical'
-    host: 'eavesdrop.openstack.org'
-    user: 'jenkins'
-    root: '/srv/yaml2ical'
-
-  - name: 'static.openstack.org'
-    host: 'static.openstack.org'
-    user: 'jenkins'
-    root: '/srv/static'
-
-  - name: 'afs-docs'
-    root: '/afs/.openstack.org/docs'
-    keytab: '/etc/zuul-launcher.keytab'
-    user: 'service/zuul-launcher'
-
-  - name: 'afs-developer-docs'
-    root: '/afs/.openstack.org/developer-docs'
-    keytab: '/etc/zuul-launcher.keytab'
-    user: 'service/zuul-launcher'
-
-zuul_nodes: []
-
-# NOTE(pabelanger): zuulv3 settings
-zuul_connections:
-  - name: 'gerrit'
-    driver: 'gerrit'
-    server: 'review.opendev.org'
-    canonical_hostname: 'opendev.org'
-    user: 'zuul'
-    sshkey: '/var/lib/zuul/ssh/id_rsa'
-    auth_type: 'digest'
-
-  - name: 'github'
-    driver: 'github'
-
-  - name: 'googlesource'
-    driver: 'gerrit'
-    server: 'gerrit-review.googlesource.com'
-    canonical_hostname: 'gerrit.googlesource.com'
-    user: 'git-infra-root.openstack.org'
-    stream_events: 'false'
-    auth_type: 'basic'
--- a/hiera/group/zuul-scheduler.yaml
+++ b/hiera/group/zuul-scheduler.yaml
@ -1,88 +0,0 @@
---
-zuul_connections:
-  - name: 'smtp'
-    driver: 'smtp'
-    server: 'localhost'
-    port: '25'
-    default_from: 'zuul@zuul.openstack.org'
-    default_to: 'zuul.reports@zuul.openstack.org'
-
-  - name: 'gerrit'
-    driver: 'gerrit'
-    server: 'review.opendev.org'
-    canonical_hostname: 'opendev.org'
-    user: 'zuul'
-    sshkey: '/var/lib/zuul/ssh/id_rsa'
-    gitweb_url_template: 'https://opendev.org/{project.name}/commit/{sha}'
-    auth_type: 'digest'
-
-  - name: 'opendaylight'
-    driver: 'gerrit'
-    server: 'git.opendaylight.org'
-    baseurl: 'git.opendaylight.org/gerrit'
-    user: 'openstack-zuul'
-    sshkey: '/var/lib/zuul/ssh/id_rsa'
-
-  - name: 'mysql'
-    driver: 'sql'
-
-  - name: 'github'
-    driver: 'github'
-    app_key: '/etc/zuul/github.key'
-    rate_limit_logging: 'false'
-
-  - name: 'googlesource'
-    driver: 'gerrit'
-    server: 'gerrit-review.googlesource.com'
-    canonical_hostname: 'gerrit.googlesource.com'
-    user: 'git-infra-root.openstack.org'
-    stream_events: 'false'
-    auth_type: 'basic'
-
-gearman_server_ssl_cert: |
-  -----BEGIN CERTIFICATE-----
-  MIIEYTCCA0mgAwIBAgIJAKkAn3gh0LBPMA0GCSqGSIb3DQEBCwUAMIG5MQswCQYD
-  VQQGEwJVUzEOMAwGA1UECAwFVGV4YXMxDzANBgNVBAcMBkF1c3RpbjEdMBsGA1UE
-  CgwUT3BlblN0YWNrIEZvdW5kYXRpb24xFzAVBgNVBAsMDkluZnJhc3RydWN0dXJl
-  MR0wGwYDVQQDDBR6dXVsdjMub3BlbnN0YWNrLm9yZzEyMDAGCSqGSIb3DQEJARYj
-  b3BlbnN0YWNrLWluZnJhQGxpc3RzLm9wZW5zdGFjay5vcmcwHhcNMTcwNjE2MjA1
-  NDAyWhcNMjcwNjE0MjA1NDAyWjCBszELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBVRl
-  eGFzMQ8wDQYDVQQHDAZBdXN0aW4xHTAbBgNVBAoMFE9wZW5TdGFjayBGb3VuZGF0
-  aW9uMRcwFQYDVQQLDA5JbmZyYXN0cnVjdHVyZTEXMBUGA1UEAwwOZ2Vhcm1hbi5z
-  ZXJ2ZXIxMjAwBgkqhkiG9w0BCQEWI29wZW5zdGFjay1pbmZyYUBsaXN0cy5vcGVu
-  c3RhY2sub3JnMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3aMR61f/
-  LZkP/acuqiCEiSFF4GI1ViNkOSPEq0CP4HfNckeW0///x6vI/uaR4MlF8g8qNFGB
-  j2FCYRW1gEzS7TLoP3xYs4SMnvXvZRbdxcozOop506quLmlfPDF1o2GzLSQYDNXe
-  WbpYiNM+EdgBjqLz4G5DdaXMMw2zYP21kbtSxJIvrpqeW/TKBGWDI2bBH81PFb9B
-  gq1P4XxI/Aw7Ez6hApLV2D6DP7JidQUGOzvGw7LUEZjLEscQU7HH8j1qDvrM2gV4
-  FRSRrtw8Yr/erBsaNr84guEZQREqiOjr1HvMZK5o1vGb69ArWSk9b8PW+A2uxvfS
-  ukv7hvNsuCouHQIDAQABo3AwbjAJBgNVHRMEAjAAMCEGCWCGSAGG+EIBDQQUFhJj
-  bGllbnQgY2VydGlmaWNhdGUwHQYDVR0OBBYEFImAuHnbfxpEEZwiiro9KEa8YA+1
-  MB8GA1UdIwQYMBaAFFP8JfdXPn8mhZLaXMa8NQIJlmneMA0GCSqGSIb3DQEBCwUA
-  A4IBAQBTNIVB758W+wBtCMlIRFUPBiR+w+7RRsY8HXME5unvO65PcsfLKQXOr3i/
-  K2SliyyBliwKY+wtbvQZVltpBiloDqslSMD6veb5YsZDzTZ+x8xP1GEhcB3c6CsN
-  0RDJ/xUGv2IXgQW8kw+MINILr9iQA6fn9dBN0OqimlchPHtvA9gO7Rv+IV3zZP+Q
-  yNWoBiZ6H5ANIt6vfcK0BHGDB6GXN9f1gpgsJd3l3vs3t/FgP1qYJiDd5VvcOXxt
-  uJziOvdg7jte0u609MWj3DOdey4HsxlEU27w13kzGI6RpPquvl/YB8Y6WMAIL8in
-  1GRv9pIfENRRHOiC57p0RSQZZ/2V
-  -----END CERTIFICATE-----
-
-zuul_ssl_cert_file_contents: |
-  -----BEGIN CERTIFICATE-----
-  MIICzjCCAbagAwIBAgIJAMV1mxY+iSJpMA0GCSqGSIb3DQEBCwUAMB8xHTAbBgNV
-  BAMMFHp1dWx2My5vcGVuc3RhY2sub3JnMB4XDTE3MDYwMjE5MzUwMloXDTI3MDUz
-  MTE5MzUwMlowHzEdMBsGA1UEAwwUenV1bHYzLm9wZW5zdGFjay5vcmcwggEiMA0G
-  CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDvgAf85YVjjBTHYJnIx8VA1VvSAidD
-  LHp2Yn+7DgUfHXjNdpftTgvWxnzXMFaglNzrNrixGNlkg1sdGDJ+DB/mvptKJUEH
-  WMfOVI98Eo0dx5w+lcP8XGTg6/SY59+PiqNpCmi+T49axQO2XKNlt+ZJsSVaEhEj
-  E2OrkZY+A8RFj07TUjSMv/pmo3AxgVjFoWszDT8pj30CTT3lg3eXXJwlqrH/P9IQ
-  FnwRSt3sR60ahFFJnvHdL1FJl/I0W5nWD6LNEpX7ryaIUIqMhQpQjGDpvG77ntfW
-  A5zhBVWPC7p2k6OaUD6AjlPMJLZh5YbyGaRN4l2Z4oizBGjoq1Qv9QehAgMBAAGj
-  DTALMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEBAOFIxTTiw10jWRKQuRKU
-  KskncSNj3ZxSjwPTOQs++hLjYYYlKA4LbWwokp7u5rTpJP/NHYLHXIda6l/Ne3JG
-  +Mby/vu0TKMX2z+0IQx3MZG7b+4NkH4jg40Q+Y879n0jvOfBplHtJB1UmQYk51fs
-  Hbrb6vvxeLRJ74JZX6t756gZnagzAoLj7DtmTfruUVjD/kRJK8gUCyKMNvN6PH3u
-  5Ls4WwOME+bFdFcxBJjj1LSKGlZoE22mSVlRqHvVXVfM9XTolvw5PequFhiPXYyj
-  ESN9QfRuVeKltTl8NdDgwlYjBBUYR5omuX5LLWUSXuvQK/dYM4ahERf3ivbXMjhF
-  M+Q=
-  -----END CERTIFICATE-----
--- a/inventory/groups.yaml
+++ b/inventory/groups.yaml
@ -156,9 +156,6 @@ groups:
    - translate[0-9]*.open*.org
    - wiki-dev[0-9]*.openstack.org
    - wiki[0-9]*.openstack.org
-    - ze[0-9]*.open*.org
-    - zm[0-9]*.open*.org
-    - zuul[0-9]*.open*.org
  puppet4:
    - afs[0-9]*.open*.org
    - afsdb[0-9]*.open*.org
@ -200,9 +197,6 @@ groups:
    - translate-dev[0-9]*.open*.org
    - wiki[0-9]*.openstack.org
    - wiki-dev[0-9]*.openstack.org
-    - ze[0-9]*.open*.org
-    - zm[0-9]*.open*.org
-    - zuul01.open*.org
  refstack:
    - refstack*.open*.org
  registry:
@ -261,6 +255,10 @@ groups:
    - wiki-dev[0-9]*.openstack.org
  zookeeper:
    - zk[0-9]*.open*.org
+  zuul:
+    - ze[0-9]*.open*.org
+    - zm[0-9]*.open*.org
+    - zuul[0-9]*.open*.org
  zuul-executor:
    - ze[0-9]*.open*.org
  zuul-merger:
@ -269,3 +267,5 @@ groups:
    - zp[0-9]*.open*.org
  zuul-scheduler:
    - zuul[0-9]*.open*.org
+  zuul-web:
+    - zuul[0-9]*.open*.org
--- a/manifests/site.pp
+++ b/manifests/site.pp
@ -497,327 +497,6 @@ node /^nb\d+\.open.*\.org$/ {
  }
 }

-# Node-OS: xenial
-node /^ze\d+\.open.*\.org$/ {
-  $group = "zuul-executor"
-
-  $gerrit_server           = 'review.opendev.org'
-  $gerrit_user             = 'zuul'
-  $gerrit_ssh_host_key     = hiera('gerrit_ssh_rsa_pubkey_contents')
-  $gerrit_ssh_private_key  = hiera('gerrit_ssh_private_key_contents')
-  $zuul_ssh_private_key    = hiera('zuul_ssh_private_key_contents')
-  $zuul_static_private_key = hiera('jenkins_ssh_private_key_contents')
-  $git_email               = 'zuul@openstack.org'
-  $git_name                = 'OpenStack Zuul'
-  $revision                = 'master'
-
-  class { 'openstack_project::server':
-    afs                       => true,
-  }
-
-  class { '::project_config':
-    url => 'https://opendev.org/openstack/project-config',
-  }
-
-  # We use later HWE kernels for better memory managment, requiring an
-  # updated AFS version which we install from our custom ppa.
-  include ::apt
-  apt::ppa { 'ppa:openstack-ci-core/openafs-amd64-hwe': }
-  package { 'linux-generic-hwe-16.04':
-    ensure  => present,
-    require => [
-      Apt::Ppa['ppa:openstack-ci-core/openafs-amd64-hwe'],
-      Class['apt::update'],
-    ],
-  }
-
-  # Skopeo is required for pushing/pulling from the intermediate
-  # registry, and is available in the projectatomic ppa.
-
-  apt::ppa { 'ppa:projectatomic/ppa': }
-  package { 'skopeo':
-    ensure  => present,
-    require => [
-      Apt::Ppa['ppa:projectatomic/ppa'],
-      Class['apt::update'],
-    ],
-  }
-
-  # Socat is also required for pushing/pulling images
-  package { 'socat':
-    ensure  => present,
-    require => [
-      Class['apt::update'],
-    ],
-  }
-
-  # NOTE(pabelanger): We call ::zuul directly, so we can override all in one
-  # settings.
-  class { '::zuul':
-    gearman_server           => 'zuul01.openstack.org',
-    gerrit_server            => $gerrit_server,
-    gerrit_user              => $gerrit_user,
-    zuul_ssh_private_key     => $gerrit_ssh_private_key,
-    git_email                => $git_email,
-    git_name                 => $git_name,
-    worker_private_key_file  => '/var/lib/zuul/ssh/nodepool_id_rsa',
-    revision                 => $revision,
-    python_version           => 3,
-    zookeeper_hosts          => 'zk01.openstack.org:2181,zk02.openstack.org:2181,zk03.openstack.org:2181',
-    zuulv3                   => true,
-    connections              => hiera('zuul_connections', []),
-    connection_secrets      => hiera('zuul_connection_secrets', []),
-    gearman_client_ssl_cert  => hiera('gearman_client_ssl_cert'),
-    gearman_client_ssl_key   => hiera('gearman_client_ssl_key'),
-    gearman_ssl_ca           => hiera('gearman_ssl_ca'),
-    #TODO(pabelanger): Add openafs role for zuul-jobs to setup /etc/openafs
-    # properly. We need to revisting this post Queens PTG.
-    trusted_ro_paths         => ['/etc/openafs', '/etc/ssl/certs', '/var/lib/zuul/ssh'],
-    trusted_rw_paths         => ['/afs'],
-    untrusted_ro_paths       => ['/etc/ssl/certs'],
-    disk_limit_per_job       => 5000,  # Megabytes
-    site_variables_yaml_file => $::project_config::zuul_site_variables_yaml,
-    require                  => $::project_config::config_dir,
-    statsd_host              => 'graphite.opendev.org',
-  }
-
-  class { '::zuul::executor': }
-
-  # This is used by the log job submission playbook which runs under
-  # python2
-  package { 'gear':
-    ensure   => latest,
-    provider => openstack_pip,
-    require  => Class['pip'],
-  }
-
-  file { '/var/lib/zuul/ssh/nodepool_id_rsa':
-    owner   => 'zuul',
-    group   => 'zuul',
-    mode    => '0400',
-    require => File['/var/lib/zuul/ssh'],
-    content => $zuul_ssh_private_key,
-  }
-
-  file { '/var/lib/zuul/ssh/static_id_rsa':
-    owner   => 'zuul',
-    group   => 'zuul',
-    mode    => '0400',
-    require => File['/var/lib/zuul/ssh'],
-    content => $zuul_static_private_key,
-  }
-
-  class { '::zuul::known_hosts':
-    known_hosts_content => "[review.opendev.org]:29418,[review.openstack.org]:29418,[104.130.246.32]:29418,[2001:4800:7819:103:be76:4eff:fe04:9229]:29418 ${gerrit_ssh_host_key}\n[git.opendaylight.org]:29418,[52.35.122.251]:29418,[2600:1f14:421:f500:7b21:2a58:ab0a:2d17]:29418 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAyRXyHEw/P1iZr/fFFzbodT5orVV/ftnNRW59Zh9rnSY5Rmbc9aygsZHdtiWBERVVv8atrJSdZool75AglPDDYtPICUGWLR91YBSDcZwReh5S9es1dlQ6fyWTnv9QggSZ98KTQEuE3t/b5SfH0T6tXWmrNydv4J2/mejKRRLU2+oumbeVN1yB+8Uau/3w9/K5F5LgsDDzLkW35djLhPV8r0OfmxV/cAnLl7AaZlaqcJMA+2rGKqM3m3Yu+pQw4pxOfCSpejlAwL6c8tA9naOvBkuJk+hYpg5tDEq2QFGRX5y1F9xQpwpdzZROc5hdGYntM79VMMXTj+95dwVv/8yTsw==\n",
-  }
-}
-
-# Node-OS: xenial
-node /^zuul\d+\.open.*\.org$/ {
-  $group = "zuul-scheduler"
-  $gerrit_server        = 'review.opendev.org'
-  $gerrit_user          = 'zuul'
-  $gerrit_ssh_host_key  = hiera('gerrit_zuul_user_ssh_key_contents')
-  $zuul_ssh_private_key = hiera('zuul_ssh_private_key_contents')
-  $zuul_url             = "http://zuul.openstack.org/p"
-  $git_email            = 'zuul@openstack.org'
-  $git_name             = 'OpenStack Zuul'
-  $revision             = 'master'
-
-  class { 'openstack_project::server': }
-
-  class { '::project_config':
-    url => 'https://opendev.org/openstack/project-config',
-  }
-
-  # NOTE(pabelanger): We call ::zuul directly, so we can override all in one
-  # settings.
-  class { '::zuul':
-    gerrit_server                 => $gerrit_server,
-    gerrit_user                   => $gerrit_user,
-    zuul_ssh_private_key          => $zuul_ssh_private_key,
-    git_email                     => $git_email,
-    git_name                      => $git_name,
-    revision                      => $revision,
-    python_version                => 3,
-    zookeeper_hosts               => 'zk01.openstack.org:2181,zk02.openstack.org:2181,zk03.openstack.org:2181',
-    zookeeper_session_timeout     => 40,
-    zuulv3                        => true,
-    connections                   => hiera('zuul_connections', []),
-    connection_secrets            => hiera('zuul_connection_secrets', []),
-    vhost_name                    => 'zuul.openstack.org',
-    zuul_status_url               => 'http://127.0.0.1:8001/openstack',
-    zuul_web_url                  => 'http://127.0.0.1:9000',
-    zuul_tenant_name              => 'openstack',
-    gearman_client_ssl_cert       => hiera('gearman_client_ssl_cert'),
-    gearman_client_ssl_key        => hiera('gearman_client_ssl_key'),
-    gearman_server_ssl_cert       => hiera('gearman_server_ssl_cert'),
-    gearman_server_ssl_key        => hiera('gearman_server_ssl_key'),
-    gearman_ssl_ca                => hiera('gearman_ssl_ca'),
-    proxy_ssl_cert_file_contents  => hiera('zuul_ssl_cert_file_contents'),
-    proxy_ssl_chain_file_contents => hiera('zuul_ssl_chain_file_contents'),
-    proxy_ssl_key_file_contents   => hiera('zuul_ssl_key_file_contents'),
-    statsd_host                   => 'graphite.opendev.org',
-    status_url                    => 'https://zuul.openstack.org',
-    relative_priority             => true,
-    web_root                      => 'https://zuul.opendev.org',
-  }
-
-  file { "/etc/zuul/github.key":
-    ensure  => present,
-    owner   => 'zuul',
-    group   => 'zuul',
-    mode    => '0600',
-    content => hiera('zuul_github_app_key'),
-    require => File['/etc/zuul'],
-  }
-
-  class { '::zuul::scheduler':
-    layout_dir     => $::project_config::zuul_layout_dir,
-    require        => $::project_config::config_dir,
-    python_version => 3,
-    use_mysql      => true,
-  }
-
-  class { '::zuul::web':
-    # We manage backups below
-    enable_status_backups => false,
-    vhosts => {
-      'zuul.openstack.org' => {
-        port       => 443,
-        docroot    => '/opt/zuul-web/content',
-        priority   => '50',
-        ssl        => true,
-        template   => 'zuul/zuulv3.vhost.erb',
-        vhost_name => 'zuul.openstack.org',
-      },
-      'zuul.opendev.org' => {
-        port       => 443,
-        docroot    => '/opt/zuul-web/content',
-        priority   => '40',
-        ssl        => true,
-        template   => 'zuul/zuulv3.vhost.erb',
-        vhost_name => 'zuul.opendev.org',
-      },
-      'zuul.openstack.org-http' => {
-        port       => 80,
-        docroot    => '/opt/zuul-web/content',
-        priority   => '50',
-        ssl        => false,
-        template   => 'zuul/zuulv3.vhost.erb',
-        vhost_name => 'zuul.openstack.org',
-      },
-      'zuul.opendev.org-http' => {
-        port       => 80,
-        docroot    => '/opt/zuul-web/content',
-        priority   => '40',
-        ssl        => false,
-        template   => 'zuul/zuulv3.vhost.erb',
-        vhost_name => 'zuul.opendev.org',
-      },
-    },
-    vhosts_flags => {
-      'zuul.openstack.org' => {
-        tenant_name => 'openstack',
-        ssl         => true,
-        use_le      => false,
-      },
-      'zuul.opendev.org' => {
-        tenant_name => '',
-        ssl         => true,
-        use_le      => true,
-      },
-      'zuul.openstack.org-http' => {
-        tenant_name => 'openstack',
-        ssl         => false,
-        use_le      => false,
-      },
-      'zuul.opendev.org-http' => {
-        tenant_name => '',
-        ssl         => false,
-        use_le      => false,
-      },
-    },
-    vhosts_ssl => {
-      'zuul.openstack.org' => {
-        ssl_cert_file_contents  => hiera('zuul_ssl_cert_file_contents'),
-        ssl_chain_file_contents => hiera('zuul_ssl_chain_file_contents'),
-        ssl_key_file_contents   => hiera('zuul_ssl_key_file_contents'),
-      },
-    },
-  }
-
-  zuul::status_backups { 'openstack-zuul-tenant':
-    tenant_name => 'openstack',
-    ssl         => true,
-    status_uri  => 'https://zuul.opendev.org/api/tenant/openstack/status',
-  }
-
-  zuul::status_backups { 'kata-zuul-tenant':
-    tenant_name => 'kata-containers',
-    ssl         => true,
-    status_uri  => 'https://zuul.opendev.org/api/tenant/kata-containers/status',
-  }
-
-  class { '::zuul::fingergw': }
-
-  class { '::zuul::known_hosts':
-    known_hosts_content => "[review.opendev.org]:29418,[review.openstack.org]:29418,[104.130.246.32]:29418,[2001:4800:7819:103:be76:4eff:fe04:9229]:29418 ${gerrit_ssh_host_key}\n[git.opendaylight.org]:29418,[52.35.122.251]:29418,[2600:1f14:421:f500:7b21:2a58:ab0a:2d17]:29418 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAyRXyHEw/P1iZr/fFFzbodT5orVV/ftnNRW59Zh9rnSY5Rmbc9aygsZHdtiWBERVVv8atrJSdZool75AglPDDYtPICUGWLR91YBSDcZwReh5S9es1dlQ6fyWTnv9QggSZ98KTQEuE3t/b5SfH0T6tXWmrNydv4J2/mejKRRLU2+oumbeVN1yB+8Uau/3w9/K5F5LgsDDzLkW35djLhPV8r0OfmxV/cAnLl7AaZlaqcJMA+2rGKqM3m3Yu+pQw4pxOfCSpejlAwL6c8tA9naOvBkuJk+hYpg5tDEq2QFGRX5y1F9xQpwpdzZROc5hdGYntM79VMMXTj+95dwVv/8yTsw==\n",
-  }
-
-  include bup
-  bup::site { 'rax.ord':
-    backup_user   => 'bup-zuulv3',
-    backup_server => 'backup01.ord.rax.ci.openstack.org',
-  }
-
-}
-
-# Node-OS: xenial
-node /^zm\d+.open.*\.org$/ {
-  $group = "zuul-merger"
-
-  $gerrit_server        = 'review.opendev.org'
-  $gerrit_user          = 'zuul'
-  $gerrit_ssh_host_key  = hiera('gerrit_ssh_rsa_pubkey_contents')
-  $zuul_ssh_private_key = hiera('zuulv3_ssh_private_key_contents')
-  $zuul_url             = "http://${::fqdn}/p"
-  $git_email            = 'zuul@openstack.org'
-  $git_name             = 'OpenStack Zuul'
-  $revision             = 'master'
-
-  class { 'openstack_project::server': }
-
-  # NOTE(pabelanger): We call ::zuul directly, so we can override all in one
-  # settings.
-  class { '::zuul':
-    gearman_server          => 'zuul01.openstack.org',
-    gerrit_server           => $gerrit_server,
-    gerrit_user             => $gerrit_user,
-    zuul_ssh_private_key    => $zuul_ssh_private_key,
-    git_email               => $git_email,
-    git_name                => $git_name,
-    revision                => $revision,
-    python_version          => 3,
-    zookeeper_hosts         => 'zk01.openstack.org:2181,zk02.openstack.org:2181,zk03.openstack.org:2181',
-    zuulv3                  => true,
-    connections             => hiera('zuul_connections', []),
-    connection_secrets      => hiera('zuul_connection_secrets', []),
-    gearman_client_ssl_cert => hiera('gearman_client_ssl_cert'),
-    gearman_client_ssl_key  => hiera('gearman_client_ssl_key'),
-    gearman_ssl_ca          => hiera('gearman_ssl_ca'),
-    statsd_host             => 'graphite.opendev.org',
-  }
-
-  class { 'openstack_project::zuul_merger':
-    gerrit_server        => $gerrit_server,
-    gerrit_user          => $gerrit_user,
-    gerrit_ssh_host_key  => $gerrit_ssh_host_key,
-    zuul_ssh_private_key => $zuul_ssh_private_key,
-    manage_common_zuul   => false,
-  }
-}
-
 # Node-OS: xenial
 node /^pbx\d*\.open.*\.org$/ {
  $group = "pbx"
--- a/modules/openstack_project/manifests/zuul_merger.pp
+++ b/modules/openstack_project/manifests/zuul_merger.pp
@ -1,29 +0,0 @@
-# == Class: openstack_project::zuul_merger
-#
-class openstack_project::zuul_merger(
-  $vhost_name = $::fqdn,
-  $gearman_server = '127.0.0.1',
-  $gerrit_server = '',
-  $gerrit_user = '',
-  $gerrit_ssh_host_key = '',
-  $zuul_ssh_private_key = '',
-  $zuul_url = "http://${::fqdn}/p",
-  $git_email = 'jenkins@openstack.org',
-  $git_name = 'OpenStack Jenkins',
-  $revision = 'master',
-  $manage_common_zuul = true,
-) {
-  class { 'openstackci::zuul_merger':
-    vhost_name               => $vhost_name,
-    gearman_server           => $gearman_server,
-    gerrit_server            => $gerrit_server,
-    gerrit_user              => $gerrit_user,
-    known_hosts_content      => "[review.opendev.org]:29418,[review.openstack.org]:29418,[104.130.246.32]:29418,[2001:4800:7819:103:be76:4eff:fe04:9229]:29418 ${gerrit_ssh_host_key}\n[git.opendaylight.org]:29418,[52.35.122.251]:29418,[2600:1f14:421:f500:7b21:2a58:ab0a:2d17]:29418 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAyRXyHEw/P1iZr/fFFzbodT5orVV/ftnNRW59Zh9rnSY5Rmbc9aygsZHdtiWBERVVv8atrJSdZool75AglPDDYtPICUGWLR91YBSDcZwReh5S9es1dlQ6fyWTnv9QggSZ98KTQEuE3t/b5SfH0T6tXWmrNydv4J2/mejKRRLU2+oumbeVN1yB+8Uau/3w9/K5F5LgsDDzLkW35djLhPV8r0OfmxV/cAnLl7AaZlaqcJMA+2rGKqM3m3Yu+pQw4pxOfCSpejlAwL6c8tA9naOvBkuJk+hYpg5tDEq2QFGRX5y1F9xQpwpdzZROc5hdGYntM79VMMXTj+95dwVv/8yTsw==\n",
-    zuul_ssh_private_key     => $zuul_ssh_private_key,
-    zuul_url                 => $zuul_url,
-    git_email                => $git_email,
-    git_name                 => $git_name,
-    manage_common_zuul       => $manage_common_zuul,
-    revision                 => $revision,
-  }
-}
--- a/modules/openstack_project/manifests/zuul_prod.pp
+++ b/modules/openstack_project/manifests/zuul_prod.pp
@ -1,59 +0,0 @@
-# == Class: openstack_project::zuul_prod
-#
-class openstack_project::zuul_prod(
-  $vhost_name = $::fqdn,
-  $gearman_server = '127.0.0.1',
-  $gerrit_server = '',
-  $gerrit_user = '',
-  $gerrit_ssh_host_key = '',
-  $zuul_ssh_private_key = '',
-  $url_pattern = '',
-  $zuul_url = '',
-  $status_url = 'https://zuul.openstack.org/',
-  $swift_authurl = '',
-  $swift_auth_version = '',
-  $swift_user = '',
-  $swift_key = '',
-  $swift_tenant_name = '',
-  $swift_region_name = '',
-  $swift_default_container = '',
-  $swift_default_logserver_prefix = '',
-  $swift_default_expiry = 7200,
-  $proxy_ssl_cert_file_contents = '',
-  $proxy_ssl_key_file_contents = '',
-  $proxy_ssl_chain_file_contents = '',
-  $statsd_host = '',
-  $project_config_repo = '',
-  $git_email = 'jenkins@openstack.org',
-  $git_name = 'OpenStack Jenkins',
-) {
-  class { 'openstackci::zuul_scheduler':
-    vhost_name                     => $vhost_name,
-    gearman_server                 => $gearman_server,
-    gerrit_server                  => $gerrit_server,
-    gerrit_user                    => $gerrit_user,
-    gerrit_strip_branch_ref        => 1,
-    known_hosts_content            => "review.openstack.org,104.130.159.134,2001:4800:7818:102:be76:4eff:fe05:9b12 ${gerrit_ssh_host_key}",
-    zuul_ssh_private_key           => $zuul_ssh_private_key,
-    url_pattern                    => $url_pattern,
-    zuul_url                       => $zuul_url,
-    job_name_in_report             => true,
-    status_url                     => $status_url,
-    swift_authurl                  => $swift_authurl,
-    swift_auth_version             => $swift_auth_version,
-    swift_user                     => $swift_user,
-    swift_key                      => $swift_key,
-    swift_tenant_name              => $swift_tenant_name,
-    swift_region_name              => $swift_region_name,
-    swift_default_container        => $swift_default_container,
-    swift_default_logserver_prefix => $swift_default_logserver_prefix,
-    swift_default_expiry           => $swift_default_expiry,
-    proxy_ssl_cert_file_contents   => $proxy_ssl_cert_file_contents,
-    proxy_ssl_key_file_contents    => $proxy_ssl_key_file_contents,
-    proxy_ssl_chain_file_contents  => $proxy_ssl_chain_file_contents,
-    statsd_host                    => $statsd_host,
-    project_config_repo            => $project_config_repo,
-    git_email                      => $git_email,
-    git_name                       => $git_name,
-  }
-}
--- a/playbooks/group_vars/all.yaml
+++ b/playbooks/group_vars/all.yaml
@ -190,3 +190,5 @@ iptables_snmp_v4_hosts:
 iptables_snmp_v6_hosts:
  # cacti02.openstack.org
  - 2001:4800:7821:105:be76:4eff:fe04:b9a5
+
+gerrit_ssh_rsa_pubkey_contents: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC+pCQlTAQYmCrOY6aPbvbyKQDcOCXibPNGIjnPPMuEItCS0vtRnqEBz7znWZS5Drq9yKpROh6uFF01ao2VnNjw6f+NdRNV19RWVe6mYN+qa2VrH2caLwBrKPiH0Xc/eK41D55dZU7IWwKYAw/NpiBaBfHavFwipI+rmEb68MH2hcimDdr/bji+0hkh3X+42dkNvmMdtkuCW6nKdAEhnXaHZc5SJR/EvzgRCfB8vbML13p46O9xhoJgn7ZWvMb3vaR5jxIkQwstUR36raEVhttBDEuWasWnHYbrM1zd3ooudbTEQf5vXISZKFygHyJFFqb4iQ76i+hDlb0VQKZCdaol gerrit-code-review@829f141b0fa5
--- a/playbooks/group_vars/zuul-executor.yaml
+++ b/playbooks/group_vars/zuul-executor.yaml
@ -1,3 +1,22 @@
 iptables_extra_public_tcp_ports:
  - 79
  - 7900
+zuul_connections:
+  - name: 'gerrit'
+    driver: 'gerrit'
+    server: 'review.opendev.org'
+    canonical_hostname: 'opendev.org'
+    user: 'zuul'
+    sshkey: '/var/lib/zuul/ssh/id_rsa'
+    auth_type: 'digest'
+
+  - name: 'github'
+    driver: 'github'
+
+  - name: 'googlesource'
+    driver: 'gerrit'
+    server: 'gerrit-review.googlesource.com'
+    canonical_hostname: 'gerrit.googlesource.com'
+    user: 'git-infra-root.openstack.org'
+    stream_events: 'false'
+    auth_type: 'basic'
--- a/playbooks/group_vars/zuul-merger.yaml
+++ b/playbooks/group_vars/zuul-merger.yaml
@ -1,4 +1,3 @@
---
 zuul_connections:
  - name: 'gerrit'
    driver: 'gerrit'
--- a/playbooks/group_vars/zuul-scheduler.yaml
+++ b/playbooks/group_vars/zuul-scheduler.yaml
@ -63,4 +63,89 @@ iptables_extra_allowed_hosts:
  - protocol: tcp
    port: 4730
    hostname: zm08.openstack.org
+zuul_connections:
+  - name: 'smtp'
+    driver: 'smtp'
+    server: 'localhost'
+    port: '25'
+    default_from: 'zuul@zuul.openstack.org'
+    default_to: 'zuul.reports@zuul.openstack.org'

+  - name: 'gerrit'
+    driver: 'gerrit'
+    server: 'review.opendev.org'
+    canonical_hostname: 'opendev.org'
+    user: 'zuul'
+    sshkey: '/var/lib/zuul/ssh/id_rsa'
+    gitweb_url_template: 'https://opendev.org/{project.name}/commit/{sha}'
+    auth_type: 'digest'
+
+  - name: 'opendaylight'
+    driver: 'gerrit'
+    server: 'git.opendaylight.org'
+    baseurl: 'git.opendaylight.org/gerrit'
+    user: 'openstack-zuul'
+    sshkey: '/var/lib/zuul/ssh/id_rsa'
+
+  - name: 'mysql'
+    driver: 'sql'
+
+  - name: 'github'
+    driver: 'github'
+    app_key: '/etc/zuul/github.key'
+    rate_limit_logging: 'false'
+
+  - name: 'googlesource'
+    driver: 'gerrit'
+    server: 'gerrit-review.googlesource.com'
+    canonical_hostname: 'gerrit.googlesource.com'
+    user: 'git-infra-root.openstack.org'
+    stream_events: 'false'
+    auth_type: 'basic'
+
+gearman_server_ssl_cert: |
+  -----BEGIN CERTIFICATE-----
+  MIIEYTCCA0mgAwIBAgIJAKkAn3gh0LBPMA0GCSqGSIb3DQEBCwUAMIG5MQswCQYD
+  VQQGEwJVUzEOMAwGA1UECAwFVGV4YXMxDzANBgNVBAcMBkF1c3RpbjEdMBsGA1UE
+  CgwUT3BlblN0YWNrIEZvdW5kYXRpb24xFzAVBgNVBAsMDkluZnJhc3RydWN0dXJl
+  MR0wGwYDVQQDDBR6dXVsdjMub3BlbnN0YWNrLm9yZzEyMDAGCSqGSIb3DQEJARYj
+  b3BlbnN0YWNrLWluZnJhQGxpc3RzLm9wZW5zdGFjay5vcmcwHhcNMTcwNjE2MjA1
+  NDAyWhcNMjcwNjE0MjA1NDAyWjCBszELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBVRl
+  eGFzMQ8wDQYDVQQHDAZBdXN0aW4xHTAbBgNVBAoMFE9wZW5TdGFjayBGb3VuZGF0
+  aW9uMRcwFQYDVQQLDA5JbmZyYXN0cnVjdHVyZTEXMBUGA1UEAwwOZ2Vhcm1hbi5z
+  ZXJ2ZXIxMjAwBgkqhkiG9w0BCQEWI29wZW5zdGFjay1pbmZyYUBsaXN0cy5vcGVu
+  c3RhY2sub3JnMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3aMR61f/
+  LZkP/acuqiCEiSFF4GI1ViNkOSPEq0CP4HfNckeW0///x6vI/uaR4MlF8g8qNFGB
+  j2FCYRW1gEzS7TLoP3xYs4SMnvXvZRbdxcozOop506quLmlfPDF1o2GzLSQYDNXe
+  WbpYiNM+EdgBjqLz4G5DdaXMMw2zYP21kbtSxJIvrpqeW/TKBGWDI2bBH81PFb9B
+  gq1P4XxI/Aw7Ez6hApLV2D6DP7JidQUGOzvGw7LUEZjLEscQU7HH8j1qDvrM2gV4
+  FRSRrtw8Yr/erBsaNr84guEZQREqiOjr1HvMZK5o1vGb69ArWSk9b8PW+A2uxvfS
+  ukv7hvNsuCouHQIDAQABo3AwbjAJBgNVHRMEAjAAMCEGCWCGSAGG+EIBDQQUFhJj
+  bGllbnQgY2VydGlmaWNhdGUwHQYDVR0OBBYEFImAuHnbfxpEEZwiiro9KEa8YA+1
+  MB8GA1UdIwQYMBaAFFP8JfdXPn8mhZLaXMa8NQIJlmneMA0GCSqGSIb3DQEBCwUA
+  A4IBAQBTNIVB758W+wBtCMlIRFUPBiR+w+7RRsY8HXME5unvO65PcsfLKQXOr3i/
+  K2SliyyBliwKY+wtbvQZVltpBiloDqslSMD6veb5YsZDzTZ+x8xP1GEhcB3c6CsN
+  0RDJ/xUGv2IXgQW8kw+MINILr9iQA6fn9dBN0OqimlchPHtvA9gO7Rv+IV3zZP+Q
+  yNWoBiZ6H5ANIt6vfcK0BHGDB6GXN9f1gpgsJd3l3vs3t/FgP1qYJiDd5VvcOXxt
+  uJziOvdg7jte0u609MWj3DOdey4HsxlEU27w13kzGI6RpPquvl/YB8Y6WMAIL8in
+  1GRv9pIfENRRHOiC57p0RSQZZ/2V
+  -----END CERTIFICATE-----
+zuul_ssl_cert_file_contents: |
+  -----BEGIN CERTIFICATE-----
+  MIICzjCCAbagAwIBAgIJAMV1mxY+iSJpMA0GCSqGSIb3DQEBCwUAMB8xHTAbBgNV
+  BAMMFHp1dWx2My5vcGVuc3RhY2sub3JnMB4XDTE3MDYwMjE5MzUwMloXDTI3MDUz
+  MTE5MzUwMlowHzEdMBsGA1UEAwwUenV1bHYzLm9wZW5zdGFjay5vcmcwggEiMA0G
+  CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDvgAf85YVjjBTHYJnIx8VA1VvSAidD
+  LHp2Yn+7DgUfHXjNdpftTgvWxnzXMFaglNzrNrixGNlkg1sdGDJ+DB/mvptKJUEH
+  WMfOVI98Eo0dx5w+lcP8XGTg6/SY59+PiqNpCmi+T49axQO2XKNlt+ZJsSVaEhEj
+  E2OrkZY+A8RFj07TUjSMv/pmo3AxgVjFoWszDT8pj30CTT3lg3eXXJwlqrH/P9IQ
+  FnwRSt3sR60ahFFJnvHdL1FJl/I0W5nWD6LNEpX7ryaIUIqMhQpQjGDpvG77ntfW
+  A5zhBVWPC7p2k6OaUD6AjlPMJLZh5YbyGaRN4l2Z4oizBGjoq1Qv9QehAgMBAAGj
+  DTALMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEBAOFIxTTiw10jWRKQuRKU
+  KskncSNj3ZxSjwPTOQs++hLjYYYlKA4LbWwokp7u5rTpJP/NHYLHXIda6l/Ne3JG
+  +Mby/vu0TKMX2z+0IQx3MZG7b+4NkH4jg40Q+Y879n0jvOfBplHtJB1UmQYk51fs
+  Hbrb6vvxeLRJ74JZX6t756gZnagzAoLj7DtmTfruUVjD/kRJK8gUCyKMNvN6PH3u
+  5Ls4WwOME+bFdFcxBJjj1LSKGlZoE22mSVlRqHvVXVfM9XTolvw5PequFhiPXYyj
+  ESN9QfRuVeKltTl8NdDgwlYjBBUYR5omuX5LLWUSXuvQK/dYM4ahERf3ivbXMjhF
+  M+Q=
+  -----END CERTIFICATE-----
--- a/playbooks/group_vars/zuul.yaml
+++ b/playbooks/group_vars/zuul.yaml
@ -0,0 +1,59 @@
+zuul_user_id: 10001
+zuul_group_id: 10001
+zuul_known_hosts: |
+  [review.opendev.org]:29418,[review.openstack.org]:29418,[104.130.246.32]:29418,[2001:4800:7819:103:be76:4eff:fe04:9229]:29418 {{ gerrit_ssh_rsa_pubkey_contents }}
+  [git.opendaylight.org]:29418,[52.35.122.251]:29418,[2600:1f14:421:f500:7b21:2a58:ab0a:2d17]:29418 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAyRXyHEw/P1iZr/fFFzbodT5orVV/ftnNRW59Zh9rnSY5Rmbc9aygsZHdtiWBERVVv8atrJSdZool75AglPDDYtPICUGWLR91YBSDcZwReh5S9es1dlQ6fyWTnv9QggSZ98KTQEuE3t/b5SfH0T6tXWmrNydv4J2/mejKRRLU2+oumbeVN1yB+8Uau/3w9/K5F5LgsDDzLkW35djLhPV8r0OfmxV/cAnLl7AaZlaqcJMA+2rGKqM3m3Yu+pQw4pxOfCSpejlAwL6c8tA9naOvBkuJk+hYpg5tDEq2QFGRX5y1F9xQpwpdzZROc5hdGYntM79VMMXTj+95dwVv/8yTsw==
+gearman_server: zuul01.openstack.org
+gearman_client_ssl_cert: |
+  -----BEGIN CERTIFICATE-----
+  MIIEYTCCA0mgAwIBAgIJAKkAn3gh0LBQMA0GCSqGSIb3DQEBCwUAMIG5MQswCQYD
+  VQQGEwJVUzEOMAwGA1UECAwFVGV4YXMxDzANBgNVBAcMBkF1c3RpbjEdMBsGA1UE
+  CgwUT3BlblN0YWNrIEZvdW5kYXRpb24xFzAVBgNVBAsMDkluZnJhc3RydWN0dXJl
+  MR0wGwYDVQQDDBR6dXVsdjMub3BlbnN0YWNrLm9yZzEyMDAGCSqGSIb3DQEJARYj
+  b3BlbnN0YWNrLWluZnJhQGxpc3RzLm9wZW5zdGFjay5vcmcwHhcNMTcwNjE2MjMw
+  MjQyWhcNMjcwNjE0MjMwMjQyWjCBszELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBVRl
+  eGFzMQ8wDQYDVQQHDAZBdXN0aW4xHTAbBgNVBAoMFE9wZW5TdGFjayBGb3VuZGF0
+  aW9uMRcwFQYDVQQLDA5JbmZyYXN0cnVjdHVyZTEXMBUGA1UEAwwOZ2Vhcm1hbi5j
+  bGllbnQxMjAwBgkqhkiG9w0BCQEWI29wZW5zdGFjay1pbmZyYUBsaXN0cy5vcGVu
+  c3RhY2sub3JnMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsh3qSWIp
+  w6kXS4IIPU7fPP2felHCtmZyfgKolYbq1iVafcc/EUHa1onlaM+w7OEHr68y3Qau
+  SY6ifEsUWCKJlhu+UlHGwVIZliL02+9EAZ1DDs6OtxKa7nOIkWq8P8kRex234QVd
+  y37+vV+/lDeCbLoGo5P0j51fnqy10afg2xRblmXgqeqaiJAvCmEnG9S9q9+gbisZ
+  1D2r+JtoTUMZtPY9NomvgdNuwmF5+VeO+CQepRWlA+0ysCFVgVwm++PNXETadHOj
+  mOSJxiq2u6fysZb7ctHgGuu+Ce3PVwah+kK/PEXADs7SjhJruSmL1ap2izc6kTFW
+  GSU/wkkPXtbWJwIDAQABo3AwbjAJBgNVHRMEAjAAMCEGCWCGSAGG+EIBDQQUFhJj
+  bGllbnQgY2VydGlmaWNhdGUwHQYDVR0OBBYEFKTyA6hjUY8jNxOEM5zuU7qecogX
+  MB8GA1UdIwQYMBaAFFP8JfdXPn8mhZLaXMa8NQIJlmneMA0GCSqGSIb3DQEBCwUA
+  A4IBAQAiLYckNAx7GQGCSXC92R23o181FiCePuNAgCb4QsaQkA/JopaLrn11R33Y
+  XO1C5fvsopKvcmEJKX0BJwNy41tz/rNmKXYy4hsPKYMsNgJQtYe98Mp+VHgAmtZ3
+  U0v49mUJA4YiLs/QmB6bmLknl1XjzJvbLu3gfVSGsquDXN1TcHLZy2fQlD6/D7HF
+  2Zj44Af4b2xFcZc7J/iErIj8LGHx3alkGAgdXw+SQkgzDeXC/DhrXC1jVJQQQzfU
+  /4GjbLiPBLb+QIAaBVv+iVVok22DSvMydjI4Zr89NXDWEOZc8oZ7nBf9Sv1+I0xB
+  6YQoN+t1YSx3G8AxPSZwyGlwhZo0
+  -----END CERTIFICATE-----
+gearman_ssl_ca: |
+  -----BEGIN CERTIFICATE-----
+  MIIERzCCAy+gAwIBAgIJAKkAn3gh0LBOMA0GCSqGSIb3DQEBCwUAMIG5MQswCQYD
+  VQQGEwJVUzEOMAwGA1UECAwFVGV4YXMxDzANBgNVBAcMBkF1c3RpbjEdMBsGA1UE
+  CgwUT3BlblN0YWNrIEZvdW5kYXRpb24xFzAVBgNVBAsMDkluZnJhc3RydWN0dXJl
+  MR0wGwYDVQQDDBR6dXVsdjMub3BlbnN0YWNrLm9yZzEyMDAGCSqGSIb3DQEJARYj
+  b3BlbnN0YWNrLWluZnJhQGxpc3RzLm9wZW5zdGFjay5vcmcwHhcNMTcwNjE2MjA1
+  MjA3WhcNMjAwNjE1MjA1MjA3WjCBuTELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBVRl
+  eGFzMQ8wDQYDVQQHDAZBdXN0aW4xHTAbBgNVBAoMFE9wZW5TdGFjayBGb3VuZGF0
+  aW9uMRcwFQYDVQQLDA5JbmZyYXN0cnVjdHVyZTEdMBsGA1UEAwwUenV1bHYzLm9w
+  ZW5zdGFjay5vcmcxMjAwBgkqhkiG9w0BCQEWI29wZW5zdGFjay1pbmZyYUBsaXN0
+  cy5vcGVuc3RhY2sub3JnMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA
+  zTnzmZkB/P+C0eHFmPyU8myEmubRVw2vK1aqx0Y7bFMlXAVH6CodI6r4VpS4vGPL
+  AfBGAmIZJlBuRysZHW3J6GuzhBFyBILHJX9PZkeJyHa3NU4ILDPMXAD/oWQnqlp1
+  3kYJ3xS1QWhPvaohC+Io3LErXOMp32mhrEmm3BGfWiXbV9STcseeLX6BKPdqBzaT
+  d8RFkrvsEJTTjwIJLreyrphrtXu/VS9uEMWaHj4/94lLXn8fn3CuUfs48kPDTlaw
+  vFg2lIGpfOui4s9Vhrafy1nrz1KzKHjhhnF80irrIo3kOkWaKeBuTyy7+MSx7PTi
+  5RgSoKTKyMbMA6nbCj73KQIDAQABo1AwTjAdBgNVHQ4EFgQUU/wl91c+fyaFktpc
+  xrw1AgmWad4wHwYDVR0jBBgwFoAUU/wl91c+fyaFktpcxrw1AgmWad4wDAYDVR0T
+  BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAe/6S1DWRtXwzBgwTCW7FR3IrpZzP
+  4eN3TUbJy6tvff+iY6+96WV9vyH62NU8oEn5TUqy8r+EiOchbXJq8pvlPAcwdaeC
+  a9pjJku40oVai0pncqDnF/WOiXNkW71bRs/qQtIuVwKwVm9OyizjWsQtjm4Ycpju
+  92liz5Q/ZZu+7eIufQYRr7lthgmTLCjqeS4qxiY7Y03ZLZpvEL+KVskkjPzHvzTO
+  S1Rq0t3ssb4uH78rvXj1Q/C2gVucUBE86P9AckSZtANGlmiKBnO6Lc1xQbsFyfSn
+  Xbt2g9IiP3nTEapCx/M8/Zl5M+XwK7pbQWdtwGnvGPoeFNV1sVT4iO1dLg==
+  -----END CERTIFICATE-----
--- a/playbooks/host_vars/zuul01.openstack.org
+++ b/playbooks/host_vars/zuul01.openstack.org
@ -1,3 +1,5 @@
+gearman_server: 127.0.0.1
 letsencrypt_certs:
  zuul-opendev-main:
    - zuul.opendev.org
+    - zuul.openstack.org
--- a/playbooks/roles/install-ansible/files/inventory_plugins/test-fixtures/results.yaml
+++ b/playbooks/roles/install-ansible/files/inventory_plugins/test-fixtures/results.yaml
@ -68,8 +68,7 @@ results:

  ze01.openstack.org:
    - afs-client
-    - puppet
-    - puppet4
+    - zuul
    - zuul-executor

  zk01.openstack.org:
--- a/playbooks/roles/zuul-executor/README.rst
+++ b/playbooks/roles/zuul-executor/README.rst
@ -0,0 +1 @@
+Run Zuul Executor
--- a/playbooks/roles/zuul-executor/files/docker-compose.yaml
+++ b/playbooks/roles/zuul-executor/files/docker-compose.yaml
@ -0,0 +1,19 @@
+# Version 2 is the latest that is supported by docker-compose in
+# Ubuntu Xenial.
+version: '2'
+
+services:
+  executor:
+    restart: always
+    image: docker.io/zuul/zuul-executor:latest
+    network_mode: host
+    user: zuul
+    volumes:
+      - /etc/zuul:/etc/zuul
+      - /opt/project-config:/opt/project-config
+      - /afs:/afs
+      - /home/zuul:/home/zuul
+      - /var/lib/zuul:/var/lib/zuul
+      - /var/log/zuul:/var/log/zuul
+      - /etc/openafs:/etc/openafs
+      - /etc/ssl/certs:/etc/ssl/certs
--- a/playbooks/roles/zuul-executor/files/logging.conf
+++ b/playbooks/roles/zuul-executor/files/logging.conf
@ -0,0 +1,49 @@
+[loggers]
+keys=root,zuul,gerrit,gear
+
+[handlers]
+keys=console,debug,normal
+
+[formatters]
+keys=simple
+
+[logger_root]
+level=WARNING
+handlers=console
+
+[logger_zuul]
+level=DEBUG
+handlers=debug,normal
+qualname=zuul
+
+[logger_gerrit]
+level=INFO
+handlers=debug,normal
+qualname=gerrit
+
+[logger_gear]
+level=WARNING
+handlers=debug,normal
+qualname=gear
+
+[handler_console]
+level=WARNING
+class=StreamHandler
+formatter=simple
+args=(sys.stdout,)
+
+[handler_debug]
+level=DEBUG
+class=logging.handlers.WatchedFileHandler
+formatter=simple
+args=('/var/log/zuul/executor-debug.log',)
+
+[handler_normal]
+level=INFO
+class=logging.handlers.WatchedFileHandler
+formatter=simple
+args=('/var/log/zuul/executor.log',)
+
+[formatter_simple]
+format=%(asctime)s %(levelname)s %(name)s: %(message)s
+datefmt=
--- a/playbooks/roles/zuul-executor/files/zuul-executor.init
+++ b/playbooks/roles/zuul-executor/files/zuul-executor.init
@ -0,0 +1,122 @@
+#! /bin/sh
+### BEGIN INIT INFO
+# Provides:          zuul-executor
+# Required-Start:    $remote_fs $syslog
+# Required-Stop:     $remote_fs $syslog
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Zuul
+# Description:       Zuul Executor
+### END INIT INFO
+
+# Do NOT "set -e"
+
+# PATH should only include /usr/* if it runs after the mountnfs.sh script
+PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/bin
+DESC="Zuul Executor"
+NAME=zuul-executor
+DAEMON=/usr/local/bin/zuul-executor
+PIDFILE=/var/run/$NAME/$NAME.pid
+SCRIPTNAME=/etc/init.d/$NAME
+USER=zuul
+
+# Exit if the package is not installed
+[ -x "$DAEMON" ] || exit 0
+
+# Read configuration variable file if it is present
+[ -r /etc/default/$NAME ] && . /etc/default/$NAME
+
+# Load the VERBOSE setting and other rcS variables
+. /lib/init/vars.sh
+
+# Define LSB log_* functions.
+# Depend on lsb-base (>= 3.0-6) to ensure that this file is present.
+. /lib/lsb/init-functions
+
+PIDFILE_DIR=$(dirname $PIDFILE)
+
+#
+# Function that starts the daemon/service
+#
+do_start()
+{
+    # Return
+    #   0 if daemon has been started
+    #   1 if daemon was already running
+    #   2 if daemon could not be started
+    #   3 if pid file already exist
+
+    if [ ! -d "$PIDFILE_DIR" ] ; then
+        mkdir -p $PIDFILE_DIR
+        chown $USER $PIDFILE_DIR
+    fi
+    ulimit -n 8192
+    ulimit -c unlimited
+    if [ -f $PIDFILE ]; then
+        return 3
+    fi
+    start-stop-daemon \
+        --start --quiet --pidfile $PIDFILE -c $USER \
+        --exec $DAEMON --test > /dev/null || return 1
+    start-stop-daemon \
+        --start --quiet --pidfile $PIDFILE -c $USER \
+        --exec $DAEMON -- $DAEMON_ARGS || return 2
+    # Add code here, if necessary, that waits for the process to be ready
+    # to handle requests from services started subsequently which depend
+    # on this one.  As a last resort, sleep for some time.
+}
+
+#
+# Function that stops the daemon/service
+#
+do_stop()
+{
+    $DAEMON stop
+    return 0
+}
+
+#
+# Function that sends a SIGHUP to the daemon/service
+#
+do_reload() {
+    $DAEMON reconfigure
+    return 0
+}
+
+case "$1" in
+    start)
+        [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME"
+        do_start
+        case "$?" in
+            0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;;
+            2) [ "$VERBOSE" != no ] && log_end_msg 1 ;;
+            3) echo "Pidfile at $PIDFILE already exists, run service zuul-executor stop to clean up."
+        esac
+        ;;
+    stop)
+        [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME"
+        do_stop
+        case "$?" in
+            0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;;
+            2) [ "$VERBOSE" != no ] && log_end_msg 1 ;;
+        esac
+        ;;
+    status)
+       status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $?
+       ;;
+    reload|force-reload)
+        #
+        # If do_reload() is not implemented then leave this commented out
+        # and leave 'force-reload' as an alias for 'restart'.
+        #
+        log_daemon_msg "Reloading $DESC" "$NAME"
+        do_reload
+        log_end_msg $?
+        ;;
+    *)
+        echo "Usage: $SCRIPTNAME {start|stop|status|force-reload}" >&2
+        exit 3
+        ;;
+esac
+
+:
--- a/playbooks/roles/zuul-executor/tasks/main.yaml
+++ b/playbooks/roles/zuul-executor/tasks/main.yaml
@ -0,0 +1,122 @@
+- name: Install PPAs
+  apt_repository:
+    repo: '{{ item }}'
+  become: yes
+  loop:
+    # For bubblewrap
+    - ppa:openstack-ci-core/bubblewrap
+    # Temporary PPA needed for bpo-27945 while waiting for SRU to be published
+    - ppa:openstack-ci-core/python-bpo-27945-backport
+    # We use later HWE kernels for better memory managment, requiring an
+    # updated AFS version which we install from our custom ppa.
+    - ppa:openstack-ci-core/openafs-amd64-hwe
+    # For skopeo
+    - ppa:projectatomic/ppa
+
+- name: Install bindep
+  pip:
+    name: bindep
+    state: present
+    executable: pip3
+  become: yes
+
+- name: Install extra packages
+  package:
+    name: '{{ item }}'
+    state: present
+  loop:
+    - jemalloc1
+    - bubblewrap
+    - skopeo
+    - socat
+
+- name: Clone zuul repo
+  git:
+    repo: https://opendev.org/zuul/zuul
+    dest: /opt/zuul
+    force: yes
+  register: zuul_repo
+
+- name: Install zuul bindep packages
+  shell:
+    cmd: apt-get install -y $(bindep -b compile)
+    chdir: /opt/zuul
+  when: zuul_repo is changed
+
+- name: Install zuul
+  shell:
+    cmd: pip install .
+    chdir: /opt/zuul
+  when: zuul_repo is changed
+
+- name: Run zuul-manage-ansible
+  shell:
+    cmd: zuul-manage-ansible
+  environment:
+    ANSIBLE_EXTRA_PACKAGES: gear
+  when: zuul_repo is changed
+
+- name: Install kubectl
+  include_role:
+    name: install-kubectl
+
+# This checks the current installed ara version with pip list and the
+# latest version of ara on pypi with pip search and if they are different
+# then we know we need to upgrade to reconcile the local version with
+# the upstream version.
+#
+# We do this using this check here rather than a pip package resource so
+# that ara's deps don't inadverdently update zuuls deps (specifically
+# ansible).
+- name: Install ARA safely
+  shell: |
+    if test $(pip3 list --format columns | sed -ne 's/^ara\s\+\([.0-9]\+\)\s\+$/\1/p') != $(pip3 search 'ara$' | sed -ne 's/^ara (\(.*\)).*$/\1/p') ; then
+      pip3 install --upgrade --upgrade-strategy=only-if-needed "ara<1.0.0"
+    fi
+
+- name: Create Zuul Executor directories
+  file:
+    state: directory
+    path: '{{ item }}'
+    owner: zuul
+    group: zuul
+  loop:
+    - /var/lib/zuul/builds
+    - /var/lib/zuul/git
+
+- name: Set up cron job to pack git refs
+  cron:
+    name: pack-git-refs
+    state: present
+    job: 'find /var/lib/zuul/git/ -maxdepth 3 -type d -name ".git" -exec git --git-dir="{}" pack-refs --all \;'
+    minute: 7
+    hour: 4
+
+- name: Install logging config
+  copy:
+    src: logging.conf
+    dest: /etc/zuul/executor-logging.conf
+
+- name: Rotate executor logs
+  include_role:
+    name: logrotate
+  vars:
+    logrotate_file_name: /var/log/zuul/executor.log
+
+- name: Rotate executor debug logs
+  include_role:
+    name: logrotate
+  vars:
+    logrotate_file_name: /var/log/zuul/executor-debug.log
+
+- name: Install init script
+  copy:
+    src: zuul-executor.init
+    dest: /etc/init.d/zuul-executor
+    mode: 0555
+  register: install_init_script
+
+- name: Register script with systemd
+  shell:
+    cmd: /bin/systemctl daemon-reload
+  when: install_init_script is changed
--- a/playbooks/roles/zuul-merger/README.rst
+++ b/playbooks/roles/zuul-merger/README.rst
@ -0,0 +1 @@
+Run zuul merger
--- a/playbooks/roles/zuul-merger/defaults/main.yaml
+++ b/playbooks/roles/zuul-merger/defaults/main.yaml
@ -0,0 +1 @@
+zuul_merger_start: false
--- a/playbooks/roles/zuul-merger/files/docker-compose.yaml
+++ b/playbooks/roles/zuul-merger/files/docker-compose.yaml
@ -0,0 +1,16 @@
+# Version 2 is the latest that is supported by docker-compose in
+# Ubuntu Xenial.
+version: '2'
+
+services:
+  merger:
+    restart: always
+    image: docker.io/zuul/zuul-merger:latest
+    network_mode: host
+    user: zuul
+    volumes:
+      - /etc/zuul:/etc/zuul
+      - /opt/project-config:/opt/project-config
+      - /home/zuul:/home/zuul
+      - /var/lib/zuul:/var/lib/zuul
+      - /var/log/zuul:/var/log/zuul
--- a/playbooks/roles/zuul-merger/files/logging.conf
+++ b/playbooks/roles/zuul-merger/files/logging.conf
@ -0,0 +1,49 @@
+[loggers]
+keys=root,zuul,gerrit,gear
+
+[handlers]
+keys=console,debug,normal
+
+[formatters]
+keys=simple
+
+[logger_root]
+level=WARNING
+handlers=console
+
+[logger_zuul]
+level=DEBUG
+handlers=debug,normal
+qualname=zuul
+
+[logger_gerrit]
+level=INFO
+handlers=debug,normal
+qualname=gerrit
+
+[logger_gear]
+level=WARNING
+handlers=debug,normal
+qualname=gear
+
+[handler_console]
+level=WARNING
+class=StreamHandler
+formatter=simple
+args=(sys.stdout,)
+
+[handler_debug]
+level=DEBUG
+class=logging.handlers.WatchedFileHandler
+formatter=simple
+args=('/var/log/zuul/merger-debug.log',)
+
+[handler_normal]
+level=INFO
+class=logging.handlers.WatchedFileHandler
+formatter=simple
+args=('/var/log/zuul/merger.log',)
+
+[formatter_simple]
+format=%(asctime)s %(levelname)s %(name)s: %(message)s
+datefmt=
--- a/playbooks/roles/zuul-merger/tasks/main.yaml
+++ b/playbooks/roles/zuul-merger/tasks/main.yaml
@ -0,0 +1,52 @@
+- name: Create Zuul directories
+  file:
+    state: directory
+    path: '{{ item }}'
+    owner: zuul
+    group: zuul
+  loop:
+    - /var/lib/zuul/git
+
+- name: Set up cron job to pack git refs
+  cron:
+    name: pack-git-refs
+    state: present
+    job: 'find /var/lib/zuul/git/ -maxdepth 3 -type d -name ".git" -exec git --git-dir="{}" pack-refs --all \;'
+    minute: 7
+    hour: 4
+
+- name: Install logging config
+  copy:
+    src: logging.conf
+    dest: /etc/zuul/merger-logging.conf
+
+- name: Rotate merger logs
+  include_role:
+    name: logrotate
+  vars:
+    logrotate_file_name: /var/log/zuul/merger.log
+
+- name: Rotate merger debug logs
+  include_role:
+    name: logrotate
+  vars:
+    logrotate_file_name: /var/log/zuul/merger-debug.log
+
+- name: Make docker-compose directory
+  file:
+    state: directory
+    path: /etc/zuul-merger
+
+- name: Install docker-compose file
+  copy:
+    src: docker-compose.yaml
+    dest: /etc/zuul-merger/docker-compose.yaml
+
+- name: Run docker-compose pull
+  shell:
+    cmd: docker-compose pull
+    chdir: /etc/zuul-merger
+
+- name: Start containers
+  include_tasks: start.yaml
+  when: zuul_merger_start | bool
--- a/playbooks/roles/zuul-merger/tasks/start.yaml
+++ b/playbooks/roles/zuul-merger/tasks/start.yaml
@ -0,0 +1,8 @@
+- name: Run docker-compose up
+  shell:
+    cmd: docker-compose up -d
+    chdir: /etc/zuul-merger
+
+- name: Run docker prune to cleanup unneeded images
+  shell:
+    cmd: docker image prune -f
--- a/playbooks/roles/zuul-scheduler/README.rst
+++ b/playbooks/roles/zuul-scheduler/README.rst
@ -0,0 +1 @@
+Run Zuul Scheduler
--- a/playbooks/roles/zuul-scheduler/files/docker-compose.yaml
+++ b/playbooks/roles/zuul-scheduler/files/docker-compose.yaml
@ -0,0 +1,16 @@
+# Version 2 is the latest that is supported by docker-compose in
+# Ubuntu Xenial.
+version: '2'
+
+services:
+  scheduler:
+    restart: always
+    image: docker.io/zuul/zuul-scheduler:latest
+    network_mode: host
+    user: zuul
+    volumes:
+      - /etc/zuul:/etc/zuul
+      - /opt/project-config:/opt/project-config
+      - /home/zuul:/home/zuul
+      - /var/lib/zuul:/var/lib/zuul
+      - /var/log/zuul:/var/log/zuul
--- a/playbooks/roles/zuul-scheduler/files/gearman-logging.conf
+++ b/playbooks/roles/zuul-scheduler/files/gearman-logging.conf
@ -0,0 +1,33 @@
+[loggers]
+keys=root,gear
+
+[handlers]
+keys=console,normal
+
+[formatters]
+keys=simple
+
+[logger_root]
+level=WARNING
+handlers=console
+
+[logger_gear]
+level=DEBUG
+handlers=normal
+qualname=gear
+
+[handler_console]
+level=WARNING
+class=StreamHandler
+formatter=simple
+args=(sys.stdout,)
+
+[handler_normal]
+level=WARNING
+class=logging.handlers.WatchedFileHandler
+formatter=simple
+args=('/var/log/zuul/gearman-server.log',)
+
+[formatter_simple]
+format=%(asctime)s %(levelname)s %(name)s: %(message)s
+datefmt=
--- a/playbooks/roles/zuul-scheduler/files/logging.conf
+++ b/playbooks/roles/zuul-scheduler/files/logging.conf
@ -0,0 +1,64 @@
+[loggers]
+keys=root,zuul,gerrit,gerrit_io,gear,kazoo,github_io
+
+[handlers]
+keys=console,debug,normal
+
+[formatters]
+keys=simple
+
+[logger_root]
+level=WARNING
+handlers=console
+
+[logger_zuul]
+level=DEBUG
+handlers=debug,normal
+qualname=zuul
+
+[logger_gerrit]
+level=INFO
+handlers=debug,normal
+qualname=gerrit
+
+[logger_gerrit_io]
+level=INFO
+handlers=debug,normal
+qualname=zuul.GerritConnection.io
+
+[logger_gear]
+level=WARNING
+handlers=debug,normal
+qualname=gear
+
+[logger_kazoo]
+level=INFO
+handlers=debug,normal
+qualname=kazoo
+
+[logger_github_io]
+level=DEBUG
+handlers=debug,normal
+qualname=github3
+
+[handler_console]
+level=WARNING
+class=StreamHandler
+formatter=simple
+args=(sys.stdout,)
+
+[handler_debug]
+level=DEBUG
+class=logging.handlers.WatchedFileHandler
+formatter=simple
+args=('/var/log/zuul/debug.log',)
+
+[handler_normal]
+level=INFO
+class=logging.handlers.WatchedFileHandler
+formatter=simple
+args=('/var/log/zuul/zuul.log',)
+
+[formatter_simple]
+format=%(asctime)s %(levelname)s %(name)s: %(message)s
+datefmt=
--- a/playbooks/roles/zuul-scheduler/handlers/main.yaml
+++ b/playbooks/roles/zuul-scheduler/handlers/main.yaml
@ -0,0 +1,4 @@
+- name: Reload Zuul Scheduler
+  shell:
+    cmd: docker-compose kill -s HUP scheduler
+    chdir: /etc/zuul-scheduler
--- a/playbooks/roles/zuul-scheduler/tasks/main.yaml
+++ b/playbooks/roles/zuul-scheduler/tasks/main.yaml
@ -0,0 +1,73 @@
+- name: Copy main.yaml into place
+  copy:
+    remote_src: yes
+    src: /opt/project-config/zuul/main.yaml
+    dest: /etc/zuul/main.yaml
+  notify: Reload Zuul Scheduler
+
+- name: Add github key
+  copy:
+    content: '{{ zuul_github_app_key }}'
+    dest: /etc/zuul/github.key
+    owner: zuul
+    group: zuul
+    mode: 0600
+
+- name: Add openstack status backup
+  include_role:
+    name: zuul-status-backup
+  vars:
+    tenant: openstack
+
+- name: Add kata status backup
+  include_role:
+    name: zuul-status-backup
+  vars:
+    tenant: kata-containers
+
+- name: Install logging config
+  copy:
+    src: logging.conf
+    dest: /etc/zuul/logging.conf
+
+- name: Install geraman logging config
+  copy:
+    src: gearman-logging.conf
+    dest: /etc/zuul/gearman-logging.conf
+
+- name: Rotate logs
+  include_role:
+    name: logrotate
+  vars:
+    logrotate_file_name: /var/log/zuul/zuul.log
+
+- name: Rotate zuul debug logs
+  include_role:
+    name: logrotate
+  vars:
+    logrotate_file_name: /var/log/zuul/zuul-debug.log
+
+- name: Rotate gearman logs
+  include_role:
+    name: logrotate
+  vars:
+    logrotate_file_name: /var/log/zuul/gearman-server.log
+
+- name: Make docker-compose directory
+  file:
+    state: directory
+    path: /etc/zuul-scheduler
+
+- name: Install docker-compose file
+  copy:
+    src: docker-compose.yaml
+    dest: /etc/zuul-scheduler/docker-compose.yaml
+
+- name: Run docker-compose pull
+  shell:
+    cmd: docker-compose pull
+    chdir: /etc/zuul-scheduler
+
+- name: Run docker prune to cleanup unneeded images
+  shell:
+    cmd: docker image prune -f
--- a/playbooks/roles/zuul-status-backup/README.rst
+++ b/playbooks/roles/zuul-status-backup/README.rst
@ -0,0 +1 @@
+Backup zuul status info
--- a/playbooks/roles/zuul-status-backup/tasks/main.yaml
+++ b/playbooks/roles/zuul-status-backup/tasks/main.yaml
@ -0,0 +1,27 @@
+# Minutes, hours, days, etc are not specified here because we are
+# interested in running this *every minute*.
+# This is a mean of backing up status.json periodically in order to provide
+# a mean of restoring lost scheduler queues if need be.
+# If the status.json is unavailable for download, no new files are created.
+- name: Install cron for status backup
+  cron:
+    name: 'zuul-scheduler-status-{{ tenant }}'
+    state: present
+    user: root
+    job: |
+      timeout -k 5 10 curl https://zuul.opendev.org/api/tenant/{{ tenant }}/status -o /var/lib/zuul/backup/{{ tenant }}_status_$(date +\\%s).json 2>/dev/null
+
+# Rotate backups and keep no more than 120 files -- or 2 hours worth of
+# backup if Zuul has 100% uptime.
+# We're not basing the rotation on time because the scheduler/web service
+# could be down for an extended period of time.
+# This is run hourly so technically up to ~3 hours worth of backups will
+# be kept.
+- name: Clean up old status backups
+  cron:
+    name: 'zuul-scheduler-status-prune-{{ tenant }}'
+    state: present
+    user: root
+    minute: 0
+    job: |
+      flock -n /var/run/{{ tenant }}_status_prune.lock ls -dt -1 /var/lib/zuul/backup/{{ tenant }}_* |sed -e '1,120d' |xargs rm -f
--- a/playbooks/roles/zuul-web/README.rst
+++ b/playbooks/roles/zuul-web/README.rst
@ -0,0 +1 @@
+Run zuul-web and zuul-fingergw
--- a/playbooks/roles/zuul-web/defaults/main.yaml
+++ b/playbooks/roles/zuul-web/defaults/main.yaml
@ -0,0 +1 @@
+zuul_web_start: false
--- a/playbooks/roles/zuul-web/files/docker-compose.yaml
+++ b/playbooks/roles/zuul-web/files/docker-compose.yaml
@ -0,0 +1,26 @@
+# Version 2 is the latest that is supported by docker-compose in
+# Ubuntu Xenial.
+version: '2'
+
+services:
+  web:
+    restart: always
+    image: docker.io/zuul/zuul-web:latest
+    network_mode: host
+    user: zuul
+    volumes:
+      - /etc/zuul:/etc/zuul
+      - /home/zuul:/home/zuul
+      - /var/lib/zuul:/var/lib/zuul
+      - /var/log/zuul:/var/log/zuul
+  fingergw:
+    restart: always
+    image: docker.io/zuul/zuul-fingergw:latest
+    network_mode: host
+    # fingergw needs to run as root so it can
+    # grab the finger port and then drop privs
+    volumes:
+      - /etc/zuul:/etc/zuul
+      - /home/zuul:/home/zuul
+      - /var/lib/zuul:/var/lib/zuul
+      - /var/log/zuul:/var/log/zuul
--- a/playbooks/roles/zuul-web/files/fingergw-logging.conf
+++ b/playbooks/roles/zuul-web/files/fingergw-logging.conf
@ -0,0 +1,49 @@
+[loggers]
+keys=root,zuul,gerrit,gear
+
+[handlers]
+keys=console,debug,normal
+
+[formatters]
+keys=simple
+
+[logger_root]
+level=WARNING
+handlers=console
+
+[logger_zuul]
+level=DEBUG
+handlers=debug,normal
+qualname=zuul
+
+[logger_gerrit]
+level=INFO
+handlers=debug,normal
+qualname=gerrit
+
+[logger_gear]
+level=WARNING
+handlers=debug,normal
+qualname=gear
+
+[handler_console]
+level=WARNING
+class=StreamHandler
+formatter=simple
+args=(sys.stdout,)
+
+[handler_debug]
+level=DEBUG
+class=logging.handlers.WatchedFileHandler
+formatter=simple
+args=('/var/log/zuul/fingergw-debug.log',)
+
+[handler_normal]
+level=INFO
+class=logging.handlers.WatchedFileHandler
+formatter=simple
+args=('/var/log/zuul/fingergw.log',)
+
+[formatter_simple]
+format=%(asctime)s %(levelname)s %(name)s: %(message)s
+datefmt=
--- a/playbooks/roles/zuul-web/files/logging.conf
+++ b/playbooks/roles/zuul-web/files/logging.conf
@ -0,0 +1,54 @@
+[loggers]
+keys=root,zuul,gerrit,gear,cherrypy
+
+[handlers]
+keys=console,debug,normal
+
+[formatters]
+keys=simple
+
+[logger_root]
+level=WARNING
+handlers=console
+
+[logger_zuul]
+level=DEBUG
+handlers=debug,normal
+qualname=zuul
+
+[logger_gerrit]
+level=INFO
+handlers=debug,normal
+qualname=gerrit
+
+[logger_cherrypy]
+level=WARN
+handlers=debug,normal
+qualname=cherrypy
+
+[logger_gear]
+level=WARNING
+handlers=debug,normal
+qualname=gear
+
+[handler_console]
+level=WARNING
+class=StreamHandler
+formatter=simple
+args=(sys.stdout,)
+
+[handler_debug]
+level=DEBUG
+class=logging.handlers.WatchedFileHandler
+formatter=simple
+args=('/var/log/zuul/web-debug.log',)
+
+[handler_normal]
+level=INFO
+class=logging.handlers.WatchedFileHandler
+formatter=simple
+args=('/var/log/zuul/web.log',)
+
+[formatter_simple]
+format=%(asctime)s %(levelname)s %(name)s: %(message)s
+datefmt=
--- a/playbooks/roles/zuul-web/handlers/main.yaml
+++ b/playbooks/roles/zuul-web/handlers/main.yaml
@ -0,0 +1,4 @@
+- name: zuul Reload apache2
+  service:
+    name: apache2
+    state: reloaded
--- a/playbooks/roles/zuul-web/tasks/main.yaml
+++ b/playbooks/roles/zuul-web/tasks/main.yaml
@ -0,0 +1,101 @@
+- name: Install apache2
+  apt:
+    name:
+      - apache2
+      - apache2-utils
+    state: present
+
+- name: Apache modules
+  apache2_module:
+    state: present
+    name: "{{ item }}"
+  loop:
+    - rewrite
+    - proxy
+    - proxy_http
+    - proxy_wstunnel
+    - ssl
+    - cache
+    - cache_disk
+    - headers
+
+- name: Remove old apache config
+  file:
+    state: absent
+    path: '{{ item }}'
+  loop:
+    - 40-zuul.opendev.org.conf
+    - 40-zuul.opendev.org-http.conf
+    - 50-zuul.openstack.org.conf
+    - 50-zuul.openstack.org-http.conf
+
+- name: Copy apache config
+  template:
+    src: zuul.vhost.j2
+    dest: /etc/apache2/sites-enabled/000-default.conf
+    owner: root
+    group: root
+    mode: 0644
+  notify: zuul Reload apache2
+
+- name: Copy whitelabel config
+  template:
+    src: openstack.vhost.j2
+    dest: "/etc/apache2/sites-enabled/010-openstack.conf"
+    owner: root
+    group: root
+    mode: 0644
+  notify: zuul Reload apache2
+
+- name: Install logging config
+  copy:
+    src: logging.conf
+    dest: /etc/zuul/web-logging.conf
+
+- name: Install fingergw logging config
+  copy:
+    src: fingergw-logging.conf
+    dest: /etc/zuul/fingergw-logging.conf
+
+- name: Rotate web logs
+  include_role:
+    name: logrotate
+  vars:
+    logrotate_file_name: /var/log/zuul/web.log
+
+- name: Rotate web debug logs
+  include_role:
+    name: logrotate
+  vars:
+    logrotate_file_name: /var/log/zuul/web-debug.log
+
+- name: Rotate fingergw logs
+  include_role:
+    name: logrotate
+  vars:
+    logrotate_file_name: /var/log/zuul/fingergw.log
+
+- name: Rotate fingergw debug logs
+  include_role:
+    name: logrotate
+  vars:
+    logrotate_file_name: /var/log/zuul/fingergw-debug.log
+
+- name: Make docker-compose directory
+  file:
+    state: directory
+    path: /etc/zuul-web
+
+- name: Install docker-compose file
+  copy:
+    src: docker-compose.yaml
+    dest: /etc/zuul-web/docker-compose.yaml
+
+- name: Run docker-compose pull
+  shell:
+    cmd: docker-compose pull
+    chdir: /etc/zuul-web
+
+- name: Start containers
+  include_tasks: start.yaml
+  when: zuul_web_start | bool
--- a/playbooks/roles/zuul-web/tasks/start.yaml
+++ b/playbooks/roles/zuul-web/tasks/start.yaml
@ -0,0 +1,8 @@
+- name: Run docker-compose up
+  shell:
+    cmd: docker-compose up -d
+    chdir: /etc/zuul-web
+
+- name: Run docker prune to cleanup unneeded images
+  shell:
+    cmd: docker image prune -f
--- a/playbooks/roles/zuul-web/templates/openstack.vhost.j2
+++ b/playbooks/roles/zuul-web/templates/openstack.vhost.j2
@ -0,0 +1,73 @@
+<VirtualHost *:80>
+  ServerName zuul.openstack.org
+  ServerAdmin webmaster@openstack.org
+
+  ErrorLog ${APACHE_LOG_DIR}/zuul-error.log
+
+  LogLevel warn
+
+  CustomLog ${APACHE_LOG_DIR}/zuul-access.log combined
+
+  Redirect / https://zuul.openstack.org/
+
+</VirtualHost>
+
+<IfModule mod_ssl.c>
+<VirtualHost *:443>
+  ServerName zuul.openstack.org
+  ServerAdmin webmaster@openstack.org
+
+  AllowEncodedSlashes On
+
+  ErrorLog ${APACHE_LOG_DIR}/zuul-ssl-error.log
+
+  LogLevel warn
+
+  CustomLog ${APACHE_LOG_DIR}/zuul-ssl-access.log combined
+
+  SSLEngine on
+  SSLProtocol All -SSLv2 -SSLv3
+  # Note: this list should ensure ciphers that provide forward secrecy
+  SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP
+  SSLHonorCipherOrder on
+
+  SSLCertificateFile /etc/letsencrypt-certs/zuul.opendev.org/zuul.opendev.org.cer
+  SSLCertificateKeyFile /etc/letsencrypt-certs/zuul.opendev.org/zuul.opendev.org.key
+  SSLCertificateChainFile /etc/letsencrypt-certs/zuul.opendev.org/ca.cer
+
+  BrowserMatch "MSIE [2-6]" \
+      nokeepalive ssl-unclean-shutdown \
+      downgrade-1.0 force-response-1.0
+  # MSIE 7 and newer should be able to use keepalive
+  BrowserMatch "MSIE [17-9]" ssl-unclean-shutdown
+
+  RewriteEngine on
+
+  RewriteRule ^/api/connection/(.*)$ http://127.0.0.1:9000/api/connection/$1 [P,L]
+  RewriteRule ^/api/console-stream ws://127.0.0.1:9000/api/tenant/openstack/console-stream [P,L]
+  RewriteRule ^/api/(.*)$ http://127.0.0.1:9000/api/tenant/openstack/$1 [P,L]
+  RewriteRule ^/(.*)$ http://127.0.0.1:9000/$1 [P,L]
+
+  AddOutputFilterByType DEFLATE application/json
+
+  <IfModule mod_cache.c>
+    CacheDefaultExpire 5
+    <IfModule mod_mem_cache.c>
+      # TODO: Should we cache the rest of the API too?
+      CacheEnable mem /api/status
+      # 12MByte total cache size.
+      MCacheSize 12288
+      MCacheMaxObjectCount 10
+      MCacheMinObjectSize 1
+      # 8MByte max size per cache entry
+      MCacheMaxObjectSize 8388608
+      MCacheMaxStreamingBuffer 8388608
+    </IfModule>
+    <IfModule mod_cache_disk.c>
+      CacheEnable disk /api/status
+      CacheRoot /var/cache/apache2/mod_cache_disk
+      CacheMaxFileSize 10000000
+    </IfModule>
+  </IfModule>
+</VirtualHost>
+</IfModule>
--- a/playbooks/roles/zuul-web/templates/zuul.vhost.j2
+++ b/playbooks/roles/zuul-web/templates/zuul.vhost.j2
@ -0,0 +1,71 @@
+<VirtualHost *:80>
+  ServerName zuul.opendev.org
+  ServerAdmin webmaster@openstack.org
+
+  ErrorLog ${APACHE_LOG_DIR}/zuul-error.log
+
+  LogLevel warn
+
+  CustomLog ${APACHE_LOG_DIR}/zuul-access.log combined
+
+  Redirect / https://zuul.opendev.org/
+
+</VirtualHost>
+
+<IfModule mod_ssl.c>
+<VirtualHost *:443>
+  ServerName zuul.opendev.org
+  ServerAdmin webmaster@openstack.org
+
+  AllowEncodedSlashes On
+
+  ErrorLog ${APACHE_LOG_DIR}/zuul-ssl-error.log
+
+  LogLevel warn
+
+  CustomLog ${APACHE_LOG_DIR}/zuul-ssl-access.log combined
+
+  SSLEngine on
+  SSLProtocol All -SSLv2 -SSLv3
+  # Note: this list should ensure ciphers that provide forward secrecy
+  SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP
+  SSLHonorCipherOrder on
+
+  SSLCertificateFile /etc/letsencrypt-certs/zuul.opendev.org/zuul.opendev.org.cer
+  SSLCertificateKeyFile /etc/letsencrypt-certs/zuul.opendev.org/zuul.opendev.org.key
+  SSLCertificateChainFile /etc/letsencrypt-certs/zuul.opendev.org/ca.cer
+
+  BrowserMatch "MSIE [2-6]" \
+      nokeepalive ssl-unclean-shutdown \
+      downgrade-1.0 force-response-1.0
+  # MSIE 7 and newer should be able to use keepalive
+  BrowserMatch "MSIE [17-9]" ssl-unclean-shutdown
+
+  RewriteEngine on
+
+  RewriteRule ^/api/tenant/(.*)/console-stream ws://127.0.0.1:9000/api/tenant/$1/console-stream [P,L]
+  RewriteRule ^/(.*)$ http://127.0.0.1:9000/$1 [P,L]
+
+  AddOutputFilterByType DEFLATE application/json
+
+  <IfModule mod_cache.c>
+    CacheDefaultExpire 5
+    <IfModule mod_mem_cache.c>
+      # TODO: Should we cache the rest of the API too?
+      CacheEnable mem /api/status
+      # 12MByte total cache size.
+      MCacheSize 12288
+      MCacheMaxObjectCount 10
+      MCacheMinObjectSize 1
+      # 8MByte max size per cache entry
+      MCacheMaxObjectSize 8388608
+      MCacheMaxStreamingBuffer 8388608
+    </IfModule>
+    <IfModule mod_cache_disk.c>
+      CacheEnable disk /api/status
+      CacheRoot /var/cache/apache2/mod_cache_disk
+      CacheMaxFileSize 10000000
+    </IfModule>
+  </IfModule>
+</VirtualHost>
+</IfModule>
--- a/playbooks/roles/zuul/README.rst
+++ b/playbooks/roles/zuul/README.rst
@ -0,0 +1 @@
+Install Zuul
--- a/playbooks/roles/zuul/defaults/main.yaml
+++ b/playbooks/roles/zuul/defaults/main.yaml
@ -0,0 +1 @@
+zuul_connection_secrets: []
--- a/playbooks/roles/zuul/tasks/main.yaml
+++ b/playbooks/roles/zuul/tasks/main.yaml
@ -0,0 +1,132 @@
+- name: Create Zuul Group
+  group:
+    name: zuul
+    gid: "{{ zuul_group_id }}"
+    system: yes
+
+- name: Create Zuul User
+  user:
+    name: zuul
+    uid: "{{ zuul_user_id }}"
+    comment: Zuul User
+    shell: /bin/bash
+    home: /home/zuul
+    group: zuul
+    create_home: yes
+    system: yes
+  # In order to run this in Zuul, we have to ignore errors.
+  # That's because in Zuul, the test nodes have a Zuul user.
+  failed_when: false
+
+- name: Create Zuul Config dir
+  file:
+    state: directory
+    path: /etc/zuul
+    owner: zuul
+    group: zuul
+
+- name: Create Zuul SSL dir
+  file:
+    state: directory
+    path: /etc/zuul/ssl
+    owner: zuul
+    group: zuul
+
+- name: Write Gearman SSL CA
+  copy:
+    content: "{{ gearman_ssl_ca }}"
+    dest: /etc/zuul/ssl/gearman-ca.pem
+    owner: zuul
+    group: zuul
+    mode: 0644
+
+- name: Write Gearman Client SSL Cert
+  copy:
+    content: "{{ gearman_client_ssl_cert }}"
+    dest: /etc/zuul/ssl/gearman-client.pem
+    owner: zuul
+    group: zuul
+    mode: 0644
+
+- name: Write Gearman Client SSL Key
+  when: gearman_client_ssl_key is defined
+  copy:
+    content: "{{ gearman_client_ssl_key }}"
+    dest: /etc/zuul/ssl/gearman-client.key
+    owner: zuul
+    group: zuul
+    mode: 0640
+
+- name: Write Gearman Server SSL Cert
+  when: gearman_server_ssl_cert is defined
+  copy:
+    content: "{{ gearman_server_ssl_cert }}"
+    dest: /etc/zuul/ssl/gearman-server.pem
+    owner: zuul
+    group: zuul
+    mode: 0644
+
+- name: Write Gearman Server SSL Key
+  when: gearman_server_ssl_key is defined
+  copy:
+    content: "{{ gearman_server_ssl_key }}"
+    dest: /etc/zuul/ssl/gearman-server.key
+    owner: zuul
+    group: zuul
+    mode: 0640
+
+- name: Write Zuul Conf File
+  template:
+    src: zuul.conf.j2
+    dest: /etc/zuul/zuul.conf
+    owner: zuul
+    group: zuul
+    mode: 0600
+
+- name: Create Zuul directories
+  file:
+    state: directory
+    path: '{{ item }}'
+    owner: zuul
+    group: zuul
+  loop:
+    - /var/log/zuul
+    - /var/run/zuul
+    - /var/lib/zuul
+    - /var/lib/zuul/ssh
+
+- name: Write Zuul SSH Key
+  copy:
+    dest: /var/lib/zuul/ssh/id_rsa
+    content: '{{ zuul_ssh_private_key_contents }}'
+    owner: zuul
+    group: zuul
+    mode: 0400
+
+- name: Create Zuul SSH directory
+  file:
+    state: directory
+    path: /home/zuul/.ssh
+    owner: zuul
+    group: zuul
+    mode: 0700
+
+- name: Write Known Hosts
+  copy:
+    dest: /home/zuul/.ssh/known_hosts
+    content: '{{ zuul_known_hosts }}'
+    owner: zuul
+    group: zuul
+    mode: 0600
+
+- name: Clone project-config repo
+  git:
+    repo: https://opendev.org/openstack/project-config
+    dest: /opt/project-config
+    force: yes
+
+- name: Install docker-compose
+  package:
+    name:
+      - docker-compose
+    state: present
--- a/playbooks/roles/zuul/templates/zuul.conf.j2
+++ b/playbooks/roles/zuul/templates/zuul.conf.j2
@ -0,0 +1,75 @@
+[gearman]
+server={{ gearman_server }}
+check_job_registration=true
+ssl_ca=/etc/zuul/ssl/gearman-ca.pem
+ssl_cert=/etc/zuul/ssl/gearman-client.pem
+{% if gearman_client_ssl_key is defined -%}
+ssl_key=/etc/zuul/ssl/gearman-client.key
+{% endif -%}
+
+[gearman_server]
+start=true
+log_config=/etc/zuul/gearman-logging.conf
+ssl_ca=/etc/zuul/ssl/gearman-ca.pem
+{% if gearman_server_ssl_cert is defined -%}
+ssl_cert=/etc/zuul/ssl/gearman-server.pem
+{% endif -%}
+{% if gearman_server_ssl_key is defined -%}
+ssl_key=/etc/zuul/ssl/gearman-server.key
+{% endif -%}
+
+[scheduler]
+tenant_config=/etc/zuul/main.yaml
+log_config=/etc/zuul/logging.conf
+state_dir=/var/lib/zuul
+relative_priority=true
+
+[fingergw]
+user=zuul
+
+[zookeeper]
+hosts={% for host in groups['zookeeper'] %}{{ (hostvars[host].ansible_default_ipv4.address) }}:2888:3888{% if not loop.last %},{% endif %}{% endfor %}
+session_timeout=40
+
+[statsd]
+server=graphite.opendev.org
+
+[merger]
+git_dir=/var/lib/zuul/git
+log_config=/etc/zuul/merger-logging.conf
+git_user_email=zuul@opendev.org
+git_user_name=OpenDev Zuul
+
+[executor]
+manage_ansible=false
+log_config=/etc/zuul/executor-logging.conf
+job_dir=/var/lib/zuul/builds
+variables=/opt/project-config/zuul/site-variables.yaml
+private_key_file=/var/lib/zuul/ssh/id_rsa
+trusted_ro_paths=/etc/openafs:/etc/ssl/certs:/var/lib/zuul/ssh
+trusted_rw_paths=/afs
+untrusted_ro_paths=/etc/ssl/certs
+disk_limit_per_job=5000
+
+[web]
+log_config=/etc/zuul/web-logging.conf
+listen_address=127.0.0.1
+listen_port=9000
+status_url=https://zuul.openstack.org
+root=https://zuul.opendev.org
+
+{% for connection in zuul_connections -%}
+[connection "{{ connection['name'] }}"]
+{% for key, value in connection.items() -%}
+{{ key }}={{ value }}
+{% endfor -%}
+{% for connection_secret in zuul_connection_secrets -%}
+{% if connection_secret['name'] == connection['name'] -%}
+{% for key, value in connection_secret.items() -%}
+{% if key != 'name' -%}
+{{ key }}={{ value }}
+{% endif -%}{# if key #}
+{% endfor -%}{# for key, value in connection_secret #}
+{% endif -%}{# if connection_secret['name'] #}
+{% endfor -%}{# for connection_secret #}
+{% endfor -%}{# for connection #}
--- a/playbooks/service-zuul.yaml
+++ b/playbooks/service-zuul.yaml
@ -0,0 +1,39 @@
+# We exclude !disabled because we want to run the noop task on all
+# of the hosts in the group, not just the active ones, because we're
+# pulling their hostvars from the fact cache. They don't stop being
+# zookeeper servers just because they are disabled.
+- hosts: "zookeeper"
+  tasks:
+    - name: Use the host so we have access to its hostvars
+      debug:
+        msg: "This debug statement is to get us access to hostvars"
+
+- hosts: "zuul:!disabled"
+  name: "Configure zuul servers"
+  roles:
+    - install-docker
+    - zuul
+
+- hosts: "zuul-merger:!disabled"
+  name: "Configure zuul merger"
+  roles:
+    - zuul-merger
+
+- hosts: "zuul-executor:!disabled"
+  name: "Configure zuul executor"
+  roles:
+    - role: kerberos-client
+      kerberos_realm: 'OPENSTACK.ORG'
+      kerberos_admin_server: 'kdc.openstack.org'
+      kerberos_kdcs:
+        - kdc03.openstack.org
+        - kdc04.openstack.org
+    - role: openafs-client
+      openafs_client_cache_size: "{{ afs_client_cache_size | default(10000000) }}" # 10GiB
+    - role: zuul-executor
+
+- hosts: "zuul-scheduler:!disabled"
+  name: "Configure zuul scheduler"
+  roles:
+    - zuul-scheduler
+    - zuul-web
--- a/playbooks/zuul/run-base.yaml
+++ b/playbooks/zuul/run-base.yaml
@ -56,6 +56,10 @@
        - group_vars/review-dev.yaml
        - group_vars/control-plane-clouds.yaml
        - group_vars/afs-client.yaml
+        - group_vars/zuul.yaml
+        - group_vars/zuul-merger.yaml
+        - group_vars/zuul-scheduler.yaml
+        - group_vars/zuul-web.yaml
        - host_vars/bridge.openstack.org.yaml
        - host_vars/etherpad01.opendev.org.yaml
        - host_vars/letsencrypt01.opendev.org.yaml
--- a/playbooks/zuul/templates/group_vars/review.yaml.j2
+++ b/playbooks/zuul/templates/group_vars/review.yaml.j2
@ -26,7 +26,6 @@ gerrit_ssh_rsa_key_contents: |
  pHMmNylg7j2NyL/9aLKs1NzdGBxpxVa5A4vgcr1DjoS1cuRVEiQoSkI6D6DCmENA
  Pb95AevPUxqqAKNZYsj4yDsXnmbFSHARijPWcpfkCDJmVhMFPObr4OE=
  -----END RSA PRIVATE KEY-----
-gerrit_ssh_rsa_pubkey_contents: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC+pCQlTAQYmCrOY6aPbvbyKQDcOCXibPNGIjnPPMuEItCS0vtRnqEBz7znWZS5Drq9yKpROh6uFF01ao2VnNjw6f+NdRNV19RWVe6mYN+qa2VrH2caLwBrKPiH0Xc/eK41D55dZU7IWwKYAw/NpiBaBfHavFwipI+rmEb68MH2hcimDdr/bji+0hkh3X+42dkNvmMdtkuCW6nKdAEhnXaHZc5SJR/EvzgRCfB8vbML13p46O9xhoJgn7ZWvMb3vaR5jxIkQwstUR36raEVhttBDEuWasWnHYbrM1zd3ooudbTEQf5vXISZKFygHyJFFqb4iQ76i+hDlb0VQKZCdaol gerrit-code-review@829f141b0fa5
 gerrit_project_ssh_rsa_key_contents: |
  -----BEGIN OPENSSH PRIVATE KEY-----
  b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABFwAAAAdzc2gtcn
--- a/playbooks/zuul/templates/group_vars/zuul-merger.yaml.j2
+++ b/playbooks/zuul/templates/group_vars/zuul-merger.yaml.j2
@ -0,0 +1 @@
+zuul_merger_start: true
--- a/playbooks/zuul/templates/group_vars/zuul-scheduler.yaml.j2
+++ b/playbooks/zuul/templates/group_vars/zuul-scheduler.yaml.j2
@ -0,0 +1,108 @@
+gearman_server_ssl_key: |
+  -----BEGIN RSA PRIVATE KEY-----
+  MIIJKAIBAAKCAgEAsma7rv193P35/Wg1Um3ARP66iLWc3Z3ZLAM6LVbr4AXo0Eww
+  dBh3DLkhfUYZnGSvWq9jj3DVo0cQZ+T6+/IczIbznCy871vmgYmhq/hjC0nNSGBH
+  Oij3jmGQ6TInOwxUhd+nsoxqPgqdCkmQuPj1NhmKypNWlb2IY/8H6AjnWTr2y+VU
+  AsHy/fw36UXi9unc9s905iN7o8R+hY6Zez/TJkmjvnA5vfIXZx6B23c/gmGN630u
+  vGrlEhjVyvHtLY7gQNeO/TmUDeK/qEqsGckLw2sukGjS/z7noh0mnd9H8wwIEGUE
+  QRJPOhkqkR0ZPfuqqQsXFeFGrIxGpDVC8CZ54Qqm+TN8DHcp/ml/VOuUGTMH6NGS
+  zxNOnn5LaSMY3u2Xs45iwteHl3kp1csOmwYtcD/fO2H9MC0NHYDuxFL2/wsfZ4EU
+  XFfnrINaF1ypWQQQCe6Q+qsCJbkhneCO4g0sKx9/X+nTgAXuIQkz1V6BeDc46c9X
+  Puf7P23S0ECuQC32T3yHGcRbuKzrzUvjwkL6CUHWJhjGvRPlf9jQkTFHi0oAg0pB
+  UsEjZX9JTx7spkrvMw7/ZQv79oCKIN5KBDA3asNnPjpNHRD4FCGwl+wvvAiT6xOZ
+  K82C5DsbdGEgt7c4YV1/VuNIx/RYJ1quZ+p1h86AT8wbC66M/M+XZl3JlzMCAwEA
+  AQKCAgAua7b4gLNodpm/C4ecbDx0d4fYHNG1hOZGooxX0d9Mip0a3khZXShVIjMJ
+  otz1KenLAgo4/9ZHRy2Iqzd3qXc+7Pqkr6t16QbgvAxacCZtgIWvCIZgJtrLrK2F
+  UGyO29V+hEThm9HlVOOqEpxa1UURD7JipdYI3qmHw4uuH+r69/HR+llS4l61IhT2
+  WR4Gu7GoczDq1V7NrUpyvDlJrcDmnJDD8/XCbCUUywZlMfFPnszL8uXfVz1F6Tpw
+  NWVOznehx7VIRNw2hML0KoH/r6Wk8tXJ88y7aAXj1AwBVmElaAMNKQvjVr1Q082U
+  tuqji0HL/LvEELtQGKwk/ErvrENYEOAPiTE7SNN5Kd7/UaLzOd7+2CBw9Ee3wXmR
+  /UfRuO8ykVXUNljJpWOeiaFj9CCr1+isEvqm8WtqkQYwvrHiS0Vio5Grpi/6aeNg
+  dr/n0nbm7G66k6iB0RLXVlp4x4zAbR/1+B8jZdTzJ/V7PwrzLLUV6yQW9KFpwrlJ
+  lLYWSlIJzPvXiq/ppMX3jx7ZEzvbQ/OHxjcuBQOGaHdhVU3AlBubUoCxGY+IEmG0
+  RaXnFWui2y0sNSAbSgReSb98ltzOlrj3S4Cd0JRQBhIpOOnaJTsIkueUEFg7DNkt
+  hl2D5p5HCAzDbMek83cGFpKgHm1SWgJS33N2ssGSmc4ttnndeQKCAQEA5qitQ5tm
+  9cGs4ps7k7Jbih6iQt2wbt2Qg+DUSC4RCkVtlFreMxTo5wb0BTgt0cXmTtFtLisu
+  uXUkU9DTCuUJ/uoISjMWKVK44YaNfXDuCrPztIFg7xL7RFDUxN42TkXKeg4D4nry
+  SGi9y4O49uP5i9CI5JstBlcq32FqF8r7zeo2mM4J0Wz7wPPIvTARBV2NX7NW9ivR
+  E8BPtGnyl7GHD+JcmHMB/UOJX8dCI2gTxIRYpqfFVP6xdH8TXROCUTaFEkMFXCre
+  meHEtwyaHZnVD4nO6dJXXZlRE3dRlZjyVgDpoGkcWeFCDshExvbXYpsB1x+3OoAM
+  mMEP4ZHLvxpaRQKCAQEAxgBNVE+6ckdtYlG+c3RwvatmfHau+NvwjT54G/9H1ZYq
+  SVkwpUYNIQBuMQZGsOt3BIv7VXONG3e6TJ2nOpwknlNG+HxBRnEIrlWgoNcC4+Gz
+  99VHV6M+MDsUXEoqs7kWLGj7Ot2VGL5u+1NCLqBrO/CHOWXDqHCJxKpo/zwAzMoW
+  aYfI8vxdKi86HndH8fOi31jIDaYpj9y3/HEpNe9W/YqfHo2Duib2WbKuCorSRrnA
+  j0h0uxzoqUTrZUs/tTgW9BY3NZhalGsxs7dh76A6yJ9/YOneTRwwfqHkvgEBDO++
+  3FR2HH26160im/4zZTMB6mWU7JpwUUkAFFX8Tuq/FwKCAQBsTmXVKgJFgWShnwxx
+  hL1Q9KNyTFBNLoJuOkLThbYAoasbjzNovvfBi1VHoiJ5rrg+6D2hASvWb3fYV2TR
+  Z8yywseTt7s/OhWP6DNF5KIRqn/TkTCn8bzETkQqEMFlLYYum6gdT2e2sl/0UOyo
+  GVIS4Z914Jtar9F0xHQhqfFktgZe59haWxc3egEXPJuxbkU026wIuXhaEuIaL+l2
+  ayilP8AE4XPcrTqzG4glwfgOPaq2zm5tQ46lygmYmdGGOthvQ8MfjQ2rKgTJgwRW
+  w+X0ftwGlPrq+1PDlTJc0U1xLsqExPZICeqPsGADIOLv7SMHFWBe+sNvcq/3VhNa
+  r5AVAoIBAAtjDw9vOmDCHNdPri1DoAw4ZD96L9veAjqNQikSCFaPOUVYnMSUf8LL
+  HIszOjOIhyK6zix+5bmTrCIl2u0y96QnU+iMdNCRRZeJEyDM3LywSUJSgLTYjYYG
+  j8gy97u4RD8vlmsvPRjcMtO/WQoHbcNXtN8nLBZuym2GA13SXJVqddmB1puqyczY
+  RHZmE7wlb9N6bp7iVHeSkP4yn9UbO5x/MWF3cADvprFH5lxy2V755coXt6bfJb4+
+  WW9M4ZARdrh44pnxdhwdAhG81SQLyfWpvpCbQo6atWtC8j2/HwlYbFiNfvFqhalL
+  qrbf6qLCSTTqvKLSyuzRzvBcdZMwSucCggEBAJP7yNcJBEuesZbT/V4RaKnb/SaH
+  xanuCR2JJTN1ITIuBCxoq9TcxShpaZpUtn/b+uZZrCBAdRdq+rVvxQeqwYWArTAA
+  JCk22v7blVqlkli4AxyS7YAzlPfKFdXt+sft+UTCh94/FSZcZWnrLKEwubtXq1Xp
+  XzA2mh8N+snUx3ff2SBP9Q9VdPEGRvFyOBDntXvpw1+6Yc+GnapG+wa7it/L+1X2
+  /w5mlUlWPkJ68NA7Dk783Hb/pD8fKWIDKmSBnZwOskbMNkW1y6n+BjRw6z/OFhhD
+  5olLy9u8zoajjwZ3fh//o2x7sc7esGoeFGyV2I44DM+GUzTO1AyPLsU3/xg=
+  -----END RSA PRIVATE KEY-----
+zuul_github_app_key: |
+  -----BEGIN OPENSSH PRIVATE KEY-----
+  b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABFwAAAAdzc2gtcn
+  NhAAAAAwEAAQAAAQEAy5sgOSgVh3EP7gOQ2d/RrpYZmAtxS6ukbeVqu++c8CmN5zNDZSEO
+  PLGXT5zyI7oyBg6rN4uijSECTV+iozDwD0oqfaYtuszlEVvw18QfN7H0En7dw8gbq97CT8
+  8ZH7LzugM3Y2mTIQXqriFVbzWdCggadDDipkdXHJW+Pn5y8YoUtdn8RMArvCJfrpeS0rb8
+  zxwrCgao9aKpk6bbS5Xv38XLnF4vbwlJrPTECvTQFu5Uokt1L1GJnQNyGFbtHapE8kn39Y
+  1WKUF02UlQz419dn8FEI1QQPyzSP29me9ftKuCVg/+2+KaQm5a5o5ecHTNJUNfhEssRT13
+  0hIjDvfg3wAAA9g/jB7tP4we7QAAAAdzc2gtcnNhAAABAQDLmyA5KBWHcQ/uA5DZ39Gulh
+  mYC3FLq6Rt5Wq775zwKY3nM0NlIQ48sZdPnPIjujIGDqs3i6KNIQJNX6KjMPAPSip9pi26
+  zOURW/DXxB83sfQSft3DyBur3sJPzxkfsvO6AzdjaZMhBequIVVvNZ0KCBp0MOKmR1cclb
+  4+fnLxihS12fxEwCu8Il+ul5LStvzPHCsKBqj1oqmTpttLle/fxcucXi9vCUms9MQK9NAW
+  7lSiS3UvUYmdA3IYVu0dqkTySff1jVYpQXTZSVDPjX12fwUQjVBA/LNI/b2Z71+0q4JWD/
+  7b4ppCblrmjl5wdM0lQ1+ESyxFPXfSEiMO9+DfAAAAAwEAAQAAAQB1GVbDCKa5KvF6dlqM
+  tAkoW/OEWrBiUOlUuylTxU+BYKTYX8dXFlfV2F2p0B4DJkc27KDUZV6rxFxKm8IyESc/4+
+  vkL/sFAGqOPU6bCZTat2IkcQqiWyhvBMLEm9tbO9SpGsh0SHfx+jEqzMkSGMekyVxNjwAL
+  meQj8Itl7du1xikDxjm+5NQN/iER61j9qTefVDUROW++PlhfzwyK/rZuuLsQTx7PBmStOR
+  vY/5KUP0VfVUtc2Y0KTiIv83BZA38pN3ZEdHkTuY/tN5EpWZ/Pxq7PHCjAggCm5tvwjTp4
+  /x6vxbnsvK/4h8WM3EyQhT/6wIbG2lafrIYPuJudyq6BAAAAgB00SO05vmNpg9QdPfg29P
+  qMPK7l69GBOQZ29P0f3gT/6ZIGfdORQbdX9Rg6igfXykrVqWCfBiZ3IXHmr9A7FH3GxJEH
+  4Tcoed7JOOFQXuyPYfdJAWFPJZxzR4S9VF4Qf9qPsl9dq9fZbX84wqZP4jyrenvPtL1veT
+  M6pd4iqvp1AAAAgQD3eiKuaVnSNp1lKFvTE402eho2W9NciON72RuUGD1ZwsjFBKHy9QvL
+  uf2A1BS+2QNKXfJLTFPkZTBN6/o7l14cx/3NlT4yVw8tCNUc1VIBXC650hNbSsU80ljgvf
+  yrELIx9TVC8oMUMu6ogJNy8u0aD+H3WpX47HkXXK6NEbcWUQAAAIEA0p40IPf2bSv11sx1
+  buOiXOQZspJx/qVuaziv3tYHzriAUII0KO2wT5qT4FZW2Mn0Ugao6MdT+Jin1TXOK5zSCZ
+  jMINdI057+qh1lWne6RJcF5D1XpSqFzl8exX/CtQhtwZ7LLsJlqKL3VCZXVZL66e9QtzKV
+  g15lheTP70XUSC8AAAAfbW9yZHJlZEBNYWNCb29rLUFpci5sb2NhbGRvbWFpbgECAwQ=
+  -----END OPENSSH PRIVATE KEY-----
+zuul_ssh_private_key_contents: |
+  -----BEGIN OPENSSH PRIVATE KEY-----
+  b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABFwAAAAdzc2gtcn
+  NhAAAAAwEAAQAAAQEA8cW77XbW0yeSlopsrRga0Tf42F7Fu6EUWPpn5hMoVC2la2Nma3zT
+  4oNl/Pope1Yv0hAtcfzIwHpM32ia1Rkqn7NzfOpU/G1H8e4yuRjPqvviMV6KEkFl8DfLhc
+  KUTE1PYqLuXkXi9MscVNlx41GGP+uunUTnxTfCyjN4ry6epM7MJ26db4j4gxxN2kifa35n
+  jsi0cTX+LDWHWOBp4lEjWohEdCQqAW6NiUrnCtaAbW7LHcC9ZnFeHfq49xRp56Cj/xb5XY
+  UhykN0IT34Us7TW85XdMThAhtHKjW+uu7tRcHJEFpRe6XFdC3DMlacVzB/e4rFKLa5hE/g
+  5s2btW5H0wAAA9jgazVJ4Gs1SQAAAAdzc2gtcnNhAAABAQDxxbvtdtbTJ5KWimytGBrRN/
+  jYXsW7oRRY+mfmEyhULaVrY2ZrfNPig2X8+il7Vi/SEC1x/MjAekzfaJrVGSqfs3N86lT8
+  bUfx7jK5GM+q++IxXooSQWXwN8uFwpRMTU9iou5eReL0yxxU2XHjUYY/666dROfFN8LKM3
+  ivLp6kzswnbp1viPiDHE3aSJ9rfmeOyLRxNf4sNYdY4GniUSNaiER0JCoBbo2JSucK1oBt
+  bssdwL1mcV4d+rj3FGnnoKP/FvldhSHKQ3QhPfhSztNbzld0xOECG0cqNb667u1FwckQWl
+  F7pcV0LcMyVpxXMH97isUotrmET+DmzZu1bkfTAAAAAwEAAQAAAQAKxp5mqhpPFP9ymD7Z
+  xC5FzvHXavnzL+3BPX/uAEKW5eXukGKbPdgPy317NgctSR0ehrwPzY2BDrJobrgf8Vw1/A
+  CUu7kH+zLutIgsOc/fthRR0P3kbGfHuiTnFFIZyIRWSB7JsuG3uWnM2lg6IoMSTEXfGpgd
+  8StMadjiLfjCLaM8O2Cv2bzoa+u6V4j51Rq00vCTRcO4URGNT5kU0EHTmlC4H5JToZSqdZ
+  0OBd1hBcmHdgT+CG0wzhbcDMnhQ97TFAVnAxF+KYN1mYIRHCl6mElUwC5+vduX6BI+4Fbz
+  anLSDV4BfWtpLk9IReBJMJd27Qb1sbuO+MttVnR8u8l5AAAAgHDnIKSdy+LSZM/G5wJiAN
+  1PWiIuFqfgDe/SCBJA7z60gi9Ek22VuPBg3fJIXKPQDM3Lr6F8LakDPvNIT9VSWHHqAzop
+  BZ6HELgpiCa6NOMwGhVw9yDEKZ/YDR58dCm8XOa8M/+XBe75bMYgqL50YvfFjnMPzCYmIM
+  MbdCSQV6N2AAAAgQD6+1HJXr3obJckgvbBlwUJuIj8zJmucwlHXq+OGdBRjVkpqEKiZRyC
+  ETDYtCzCtCQl6vDdI2nnuflVrEqQW7hxAU94u7ipgQv4GpNWY8Fq7AovLGnU2gYEkK9BEj
+  dcjGXb8303K0tNI9jIsgx3TDk/cGnmqjIM99FsyVQwK4iE5wAAAIEA9ptGjX6SB5KimIjm
+  1ChAhx9I+OrIC4gGUBHtsOP+LZL6UQIEi/mUMBmDLi8k14AMG9fK1zrt8HRequIAGxjxka
+  X58RKjrCY/UVW4xaMikMXZuTzq2F4KA0F5rpFD+1E00UledMWq7u1o1R1qnFEW6z/B9rUl
+  TFg6lZUdaYGinDUAAAAfbW9yZHJlZEBNYWNCb29rLUFpci5sb2NhbGRvbWFpbgECAwQ=
+  -----END OPENSSH PRIVATE KEY-----
--- a/playbooks/zuul/templates/group_vars/zuul-web.yaml.j2
+++ b/playbooks/zuul/templates/group_vars/zuul-web.yaml.j2
@ -0,0 +1 @@
+zuul_web_start: true
--- a/playbooks/zuul/templates/group_vars/zuul.yaml.j2
+++ b/playbooks/zuul/templates/group_vars/zuul.yaml.j2
@ -0,0 +1,80 @@
+gearman_client_ssl_key: |
+  -----BEGIN RSA PRIVATE KEY-----
+  MIIJKgIBAAKCAgEA1h1BKYRC6qDlGR89fKRDKTueV6MyG9YhbLoTbzc5EwtOQQTp
+  KMCTgZElSnzaWqAqfsaNVEawI9UJiJ0k+jklFRY5DmKk5pOZqwSF5QbkVj1OHQ4M
+  ctNKM6/YO3AF3K/bPxjQLj6rWNmyHzggGUCft7bxyDFapxNMLe2DjCTVodpdBv1I
+  xeydL0DHxai3Mb0H0RznBqVw04oV1TkdLcKTErvnj+YIa7PIhQDazuOVa+Dz9pte
+  LmRJxdQV6FgSD9W+amMKK234QkJcLrdO8dxzC4wlJB7Hc+HGk0HCZJF4RoUhtnT6
+  H2xqc1X//OUcjwWH8NP/TfJXKWm6tSIsdjaGPh8ymx8bsqpIlzbX4yUq1m7463r+
+  MnugYEuWosp7vnsM5x/iq4oFWmtOM3htJ07C7fUMzOp/VSEo5n03ArqBwgzwr/hr
+  VpH3KUqQQ4/WCHEsf8QIXXJCH2Ls+SOBLulamGwTrRFEwUylezZec3mXr6QyyCfH
+  gSLYmyxxMQzTSSNAfBuRkZiknSXtrUpsMXD8G1z1fXisqlTHGHb74Ugtp7LIM2q5
+  YoFWS+JmjXkyIDZCqC9W1FsXyo5gI0ghNjd/kky5JpoWfTrWMrJnYa9HYo42vSfn
+  m4KwHDxh3EXu9kzYV++h7xP1+0mTb64gNMQgg+qK7d7tId5DeSg25SJHW1kCAwEA
+  AQKCAgEAv989ap/se2ethcK6Df0BdmzHq49CMzHDiDSDf/GDwu4ptRhafLt+M+jG
+  +yZBYl8PVcZGFhS2eZXKUlNINLeK5Iein6KEVWBFn7yQ5Dk125Zabq0NOMThRMo6
+  wqDTj/1DQxrQS/C7CgcjmNhp41dHCZH2v0iDDR+875ddf/PuQXl2TfIiCcPM4/Bw
+  VU+owvi7jYgR+6G8JsUiZY4l+MDZnTsn+orQVvuoIJAwhJ/rYd4XoZF/Z6FVfuNc
+  snZh0TDgz2NrVJnalD31b6OzKgg8TEfNbL3sTIsxsPqH4il+F+vr1x6imhBEoJCb
+  spv56KyzMnw32DjoJONrfjBemZyo1FR3++VDJAIQ9Xry7AjSd2SPZHVLSi0tOOZL
+  pgeQTFo0K/svQVKLS8ypdZmt9+Osmz7B9ua5SLMkcWDSOrhZ4jZhbSg2CAUCFShW
+  G0QhYJNxXQYAGGpdqZxSQXH9cTVtBqV79ZA6L1ZDbh7EUA5BEwc4YUEX3JdQhN7S
+  u9vYXmDvIfialZi9DQ2LBqkiKuSzV9vzgm7n8pYW+KJ44MwToPdarbrL9W1v8/R5
+  Ur2I+0BQ7rFWPeL4h7PXOyrtWtdfl71OBiS+H+xQymh3dAe+7Xk8ey0BC+tPGNNG
+  YXr7DcTeYAnZF1+BOZ0XZ9hH0PM3rud+4rqEKCaZUckzHAZwOQECggEBAPWVCkQI
+  rZJcBun/OfTsN7C9fKWTRu7oL0fzCDnu5MOZYMHEY9qiSrIvGRBHoRmFXK+QbVmE
+  XNFc1CFyUl8L6tGZT6XkGnfj50d2dM+FccMqbBxg8OMMWdf2F0qD93MAvZd/Nrug
+  BWwORji29GrUf3FoRMpZbTL68yU6fOoGZU/pyMRJk9iE+dc2DEk6uBKf9RW1TBbK
+  PT7uA6KsUOJpB5WWXckWAZOR0BwlejZ8JMEdXSyctaYNDuznxLbe4osXVRhh/XVN
+  En6bW8BJblMm0eAfduEKFyWNoqs8wS9vQ4qGJiU74t0yh0liGnHGyNcU6gFuk1EM
+  cqmEY7L/AdBbbnkCggEBAN8yex7lWXPOxHpVzJ19/qZZoLBgPQ6C8QfNzCozB3du
+  wVxh4s9U10G7SP/nIZ8CdJH1lM7+v1Q1NWmVbjlp8mnU6K2QtLTl3efec+mzZB21
+  N344nKrxEubz+upJQReIa/cxMKewlWTg9a4GxZjTHEkB06xXLnBfiVW+LbXeKeiG
+  i59yTIDS2t5/rQRzPzutvkjFO39v1rwAiTIF1sqSkz3yBuqmBgh4GDaAcO+sTCeR
+  UnsWkbFFAohfD5yww2Q/Dx8au/b31mIig2MKvz4kTMQapYcV5q8Y0s2K2+/52O+B
+  ge4RWws+tBb+nKIcm6yiR0WUSVJBdcpQncHhbjRIm+ECggEBAOxN1gvy4blkTc7Z
+  JJZ0uX2aRxc3aNi3l88+nlrIcV14925bn82fvgpIYXCVzAE3nyDb8yxgvcNC9Gee
+  jn4ghHnccJRqscFNDZ1o8StB915ZMp/387I1jznL9UthQjhprQTahvrxFmaMMaue
+  9/7XrC2erBqdBAM7D71x0wKI1vGXPfUJ63Y7NgCMZDQOiVJ6kiSqR6XiQh6MffkI
+  n+fMMl0Qy/uS7j9l3f9HXJqSx/b+X0pvHCbEh+kTduiT/R7je6EzoOQ/Hh2vNhEH
+  V14xi1+CRyxxPiaHa9AjbKxM/ouLW6cWQcygMyc8e7+hDF5RJH3uPViOhsJwvlJd
+  KAyp/dkCggEBAJSMYolDl++NtBK/u/kt/Cf3Cw2YX8qit4y8GaAUamnA2wyDUZMw
+  IjvrTECVMjlERxVF346M2gZPi4cEH2Iy81Ygj+PEpaCoRLKnyXnHHWDwVUi6oPrc
+  i/oOc/cuXhYtg733jSxuSF/loV37v9Ng3jhw8NKJC61ayGq6sm2SuU27Dn5Gckhj
+  Dax8SUjm8zTjA/wm2NyOMNrbeHREkposR7c4uAXADc/hLixH++JoSB8lh0HI2Zqk
+  FXVx31AoDUNQ/N10y5kphhO2aL+oTXQscLMsEPMBTpFG8jY+rvbe0NVG2pT6FCA9
+  0VpkhxcV9z5Emy7h7JLEYoMOeJCrWs+Na8ECggEAK4tGEvMnvTq5EhcCHNgr8q8a
+  ffBkAZehG3sOO47mTfkhSa4vX23c1+Uhi40cNwxnC/Nsk8mAuIhxg6rK3vY/JW5C
+  TbTPCZqeOoA7XzHsFFtpC437egAxixKU55b0Kx+lvW0nXNBI+dRGMNK/qs19rOm/
+  sxU15CtL/jJfvQFddY+XlJjlwKhdcx1R4yq5UdHA21SMXt2QiFhV5VFXlbRHRGSO
+  dyTbQ2oR9eBf5gaS3uGRYP2Sk1gnIes1jFUPqbu4Knucrl0ChlvTSjNfyK7scuCg
+  OJM6gKyqJGxNgplf7hhTyFhSD4iSGLK9oCJL63IobjcNZYKYkCeV6Zrk7PaJpg==
+  -----END RSA PRIVATE KEY-----
+zuul_ssh_private_key_contents: |
+  -----BEGIN OPENSSH PRIVATE KEY-----
+  b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABFwAAAAdzc2gtcn
+  NhAAAAAwEAAQAAAQEA1gRq9lCu6nJdLzzs2pfQwUovQfOH13HLj8DDbc/wYwDMocjNweeu
+  A+5XYyXN83hbGaREvLz6QnYPHKaEd2mGt+nqgsTGmOVsvq7KntApKFj2RVD0ij0Ycy5iRr
+  YJY7PaT/0t2fjzMAXEpzb6OmDjCW0ftyRyzi4VNPHLeHD0GCcnuzMzi9VoLz5ELkM+QCC8
+  Jxne4QsLaLGyfAERL83lidhuEZ9qKkhLYCZ0abnA/OHU267eYRUFWAmDPK8VbdmPnbnacF
+  Cc5vcs0JJxnUKuESdM392ybZRHT2zr7O212obaQ/InKWex6QD0o0IBUHYLH3cyWeVhSFhP
+  qGAm7X4o2wAAA9iUz3FQlM9xUAAAAAdzc2gtcnNhAAABAQDWBGr2UK7qcl0vPOzal9DBSi
+  9B84fXccuPwMNtz/BjAMyhyM3B564D7ldjJc3zeFsZpES8vPpCdg8cpoR3aYa36eqCxMaY
+  5Wy+rsqe0CkoWPZFUPSKPRhzLmJGtgljs9pP/S3Z+PMwBcSnNvo6YOMJbR+3JHLOLhU08c
+  t4cPQYJye7MzOL1WgvPkQuQz5AILwnGd7hCwtosbJ8AREvzeWJ2G4Rn2oqSEtgJnRpucD8
+  4dTbrt5hFQVYCYM8rxVt2Y+dudpwUJzm9yzQknGdQq4RJ0zf3bJtlEdPbOvs7bXahtpD8i
+  cpZ7HpAPSjQgFQdgsfdzJZ5WFIWE+oYCbtfijbAAAAAwEAAQAAAQEAg9J+y68QvkmpCgKd
+  5Vqjc5stFpNZNaPa/XV/KnFtIJ4KbRBRZEE+1x8EZoaPn4qfmmCrEhHYl/09+6i5aQ/vsf
+  J7xwZLSTvvSlhBZ6bR4w9AyZs+tLNDDxcf42wWxnmuW5yXlG4Z5Jd49IIRiMnKrjCv20+x
+  AzwxRcY1TL9OKl0628rlD+3DcDlNq+VM+T7PFGKpZem3cZdUgkYPIiHBKwe3TQSt55X2mY
+  K+Ja4x6f5csO+oLvbe2wyfq4epAEwpw9DvzF9hv73rtOLtBrO9uam+9BbwLEBCzBZsvBYA
+  nCZiAf9LvJb96w70yO0NGwQN0cUlTUm5/hAHxqk2QOH+oQAAAIEA85bY18wTXotIgoceJj
+  gfzANSWaZYolZZpXXK54qFBudcXis+NUJN258guu4Pw3gnvTv3j5sJSnIq1/ZPgE+9T0Ci
+  Gu+lvQnVq464cY0swph1qOSesi1dVKV69Ah4EAgS1BCTv7A6jHZNYchiPmZa7+mNp+DRES
+  IbTbvwR1ihcRoAAACBAPkb7OW8Bnz+DDpx+xPdvGOEJHN7Lj7vV3+omS8Ax0/1vyEYPhHx
+  QRe08XmCoERIc5oSHzzqR4BxAKgdiC5VIBQn+uSu5TaPV+sihcglPgw9fe7N60x4CgU33x
+  V+4t+Vx7rdBmGA6ojHreS8chUm7PjMJcoIatD9Z766WgS+AheLAAAAgQDb7/0LU0ZyZ/PL
+  NdRLjG7o3gCqOPyyHzE0VNHQiiBoAFKXLM//QHSKNMcany5kbvkpaKzKJJMS+ohagjv/oi
+  GvKJZEWqFZS99wcHXI0Zqh4Z3vg6mhcbQlQomW+G3Ajz5wnoYqTbBEIIA14ivIklH5llAp
+  /pjwbFxlotxhK/nd8QAAAB9tb3JkcmVkQE1hY0Jvb2stQWlyLmxvY2FsZG9tYWluAQI=
+  -----END OPENSSH PRIVATE KEY-----