system-config/zuul.d/system-config-run.yaml
Ian Wienand c9215801f0 Generate ssl check list directly from letsencrypt variables
This autogenerates the list of ssl domains for the ssl-cert-check tool
directly from the letsencrypt list.

The first step is the install-certcheck role that replaces the
puppet-ssl_cert_check module that does the same.  The reason for this
is so that during gate testing we can test this on the test
bridge.openstack.org server, and avoid adding another node as a
requirement for this test.

letsencrypt-request-certs is updated to set a fact
letsencrypt_certcheck_domains for each host that is generating a
certificate.  As described in the comments, this defaults to the first
host specified for the certificate and the listening port can be
indicated (if set, this new port value is stripped when generating
certs as is not necessary for certificate generation).

The new letsencrypt-config-certcheck role runs and iterates all
letsencrypt hosts to build the final list of domains that should be
checked.  This is then extended with the
letsencrypt_certcheck_additional_domains value that covers any hosts
using certificates not provisioned by letsencrypt using this
mechanism.

These additional domains are pre-populated from the openstack.org
domains in the extant check file, minus those openstack.org domain
certificates we are generating via letsencrypt (see
letsencrypt-create-certs/handlers/main.yaml).  Additionally, we
update some of the certificate variables in host_vars that are
listening on port !443.

As mentioned, bridge.openstack.org is placed in the new certcheck
group for gate testing, so the tool and config file will be deployed
to it.  For production, cacti is added to the group, which is where
the tool currently runs.  The extant puppet installation is disabled,
pending removal in a follow-on change.

Change-Id: Idbe084f13f3684021e8efd9ac69b63fe31484606
2020-05-20 14:27:14 +10:00

742 lines
23 KiB
YAML

- job:
name: system-config-run
description: |
Run the "base" playbook for system-config hosts.
This is a parent job designed to be inherited.
abstract: true
pre-run: playbooks/zuul/run-base-pre.yaml
run: playbooks/zuul/run-base.yaml
post-run: playbooks/zuul/run-base-post.yaml
vars:
install_ansible_ara_enable: true
zuul_copy_output: "{{ copy_output | combine(host_copy_output | default({})) }}"
stage_dir: "{{ ansible_user_dir }}/zuul-output"
copy_output:
'/var/log/syslog': logs_txt
'/var/log/messages': logs_txt
'/var/log/docker': logs
host-vars:
bridge.openstack.org:
host_copy_output:
'{{ zuul.project.src_dir }}/junit.xml': logs
# Note: the following two jobs implement the variant-based multiple
# inheritance trick. Both of these variants will always apply,
# therefore both parents will appear in the inheritance hierarchy).
- job:
name: system-config-run-containers
parent: system-config-run
# Note: see above re multiple-inheritance.
- job:
name: system-config-run-containers
parent: opendev-buildset-registry-consumer
description: |
Run the "base" playbook for system-config hosts which use
containers.
This is a parent job designed to be inherited. Use this job if
the service in question is container-based. It expects a
buildset registry and pulls images from the intermediate
registry.
- job:
name: system-config-run-base
parent: system-config-run
description: |
Run the "base" playbook on each of the node types
currently in use.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: xenial
label: ubuntu-xenial
- name: bionic
label: ubuntu-bionic
- name: focal
label: ubuntu-focal
files:
- playbooks/
- roles/
- testinfra/
- job:
name: system-config-run-base-ansible-devel
parent: system-config-run-base
description: |
Run the base playbook with the latest ansible.
required-projects:
- name: github.com/ansible/ansible
override-checkout: devel
- name: github.com/philpep/testinfra
- name: openstack/openstacksdk
- name: recordsansible/ara
# NOTE(ianw): 2019-06-11 pinned to stable branch until we handle 1.0
# http://lists.openstack.org/pipermail/openstack-infra/2019-June/006400.html
override-checkout: stable/0.x
vars:
bridge_ansible_name: '{{ ansible_user_dir}}/src/github.com/ansible/ansible'
bridge_ansible_version: null
bridge_openstacksdk_name: '{{ ansible_user_dir }}/src/opendev.org/openstack/openstacksdk'
bridge_openstacksdk_version: null
bridge_ara_name: '{{ ansible_user_dir}}/src/opendev.org/recordsansible/ara'
bridge_ara_version: null
# Although we don't have an arm64 based bridge; Zuul can't currently
# allocate a mixed x86/arm64 situation across clouds. Thus it helps
# to keep this clean so we can run the other tests.
- job:
name: system-config-run-base-arm64
parent: system-config-run
description: |
Run the "base" playbook on ARM64.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic-arm64
- name: bionic
label: ubuntu-bionic-arm64
files:
- playbooks/
- roles/
- testinfra/
- job:
name: system-config-run-eavesdrop
parent: system-config-run-containers
description: |
Run the playbook for an eavesdrop server.
required-projects:
- opendev/ansible-role-puppet
- opendev/system-config
- openstack/project-config
requires: accessbot-container-image
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: eavesdrop01.openstack.org
label: ubuntu-xenial
vars:
run_playbooks:
- playbooks/service-eavesdrop.yaml
files:
- playbooks/service-eavesdrop.yaml
- playbooks/run-accessbot.yaml
- playbooks/group_vars/eavesdrop.yaml
- playbooks/group_vars/puppet.yaml
- playbooks/roles/zuul-user
- playbooks/roles/install-docker
- playbooks/roles/puppet-install/
- playbooks/roles/disable-puppet-agent/
- playbooks/roles/accessbot
- playbooks/roles/logrotate
- modules/openstack_project/manifests/eavesdrop.pp
- manifests/eavesdrop.pp
- docker/accessbot/
- testinfra/test_eavesdrop.py
- job:
name: system-config-run-codesearch
parent: system-config-run
description: |
Run the playbook for an codesearch server.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: codesearch01.openstack.org
label: ubuntu-xenial
required-projects:
- opendev/ansible-role-puppet
- opendev/system-config
- openstack/project-config
files:
- playbooks/install-ansible.yaml
- playbooks/service-codesearch.yaml
- playbooks/group_vars/puppet.yaml
- playbooks/roles/run-puppet/
- playbooks/roles/install-ansible-roles/
- playbooks/roles/sync-project-config
- playbooks/roles/puppet-install/
- playbooks/roles/disable-puppet-agent/
- modules/openstack_project/manifests/codesearch.pp
- modules/openstack_project/files/resync-hound-config.sh
- manifests/codesearch.pp
vars:
run_playbooks:
- playbooks/service-codesearch.yaml
- job:
name: system-config-run-letsencrypt
parent: system-config-run
description: |
Run the playbook for letsencrypt key acquisition
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: adns-letsencrypt.opendev.org
label: ubuntu-bionic
- name: letsencrypt01.opendev.org
label: ubuntu-bionic
- name: letsencrypt02.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-nameserver.yaml
- playbooks/service-letsencrypt.yaml
host-vars:
bridge.openstack.org:
host_copy_output:
'/var/lib/certcheck': logs
letsencrypt01.opendev.org:
host_copy_output:
'/var/log/acme.sh': logs
letsencrypt02.opendev.org:
host_copy_output:
'/var/log/acme.sh': logs
files:
- playbooks/install-ansible.yaml
- playbooks/group_vars/letsencrypt.yaml
- playbooks/roles/letsencrypt
- job:
name: system-config-run-lists
parent: system-config-run
description: |
Run the playbook for a list server.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: lists.openstack.org
label: ubuntu-xenial
required-projects:
- opendev/ansible-role-puppet
- opendev/system-config
files:
- playbooks/install-ansible.yaml
- modules/openstack_project/manifests/lists.pp
- playbooks/host_vars/lists.openstack.org.yaml
- playbooks/group_vars/puppet
- playbooks/roles/run-puppet/
- playbooks/roles/install-ansible-roles/
- roles/exim
vars:
run_playbooks:
- playbooks/remote_puppet_else.yaml
- job:
name: system-config-run-nodepool
parent: system-config-run
description: |
Run the playbook for nodepool.
timeout: 3600
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: zk01.opendev.org
label: ubuntu-bionic
- name: nl01.openstack.org
label: ubuntu-xenial
- name: nb04.opendev.org
label: ubuntu-bionic
required-projects:
- opendev/ansible-role-puppet
- opendev/system-config
- openstack/project-config
host-vars:
nl01.openstack.org:
host_copy_output:
'/etc/nodepool/nodepool.yaml': logs
nb04.opendev.org:
host_copy_output:
'/etc/nodepool/nodepool.yaml': logs
vars:
run_playbooks:
- playbooks/service-letsencrypt.yaml
- playbooks/service-zookeeper.yaml
- playbooks/service-nodepool.yaml
- playbooks/remote_puppet_else.yaml
files:
- playbooks/install-ansible.yaml
- playbooks/group_vars/nodepool.yaml
- playbooks/group_vars/nodepool-builder.yaml
- playbooks/group_vars/nodepool-launcher.yaml
- playbooks/group_vars/puppet
- playbooks/roles/run-puppet/
- playbooks/roles/install-ansible-roles/
- playbooks/roles/configure-openstacksdk/
- playbooks/roles/nodepool
- playbooks/templates/clouds/
- testinfra/test_nodepool.py
- job:
name: system-config-run-dns
parent: system-config-run
description: |
Run the playbook for dns.
required-projects:
- opendev/zone-opendev.org
- opendev/zone-zuul-ci.org
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: adns1.opendev.org
label: ubuntu-bionic
- name: ns1.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-nameserver.yaml
host-vars:
adns1.opendev.org:
host_copy_output:
'/etc/bind/named.conf': logs
'/var/lib/bind/zones': logs
files:
- playbooks/install-ansible.yaml
- playbooks/group_vars/adns.yaml
- playbooks/group_vars/dns.yaml
- playbooks/host_vars/(ad)?ns\d+.opendev.org.yaml
- playbooks/zuul/templates/group_vars/adns.yaml.j2
- playbooks/zuul/templates/group_vars/ns.yaml.j2
- playbooks/roles/master-nameserver/
- playbooks/roles/nameserver/
- testinfra/test_adns.py
- testinfra/test_ns.py
- job:
name: system-config-run-backup
parent: system-config-run
description: |
Run the playbook for backup configuration
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: backup01.region.provider.opendev.org
label: ubuntu-bionic
- name: backup-test01.opendev.org
label: ubuntu-bionic
- name: backup-test02.opendev.org
label: ubuntu-xenial
vars:
run_playbooks:
- playbooks/service-backup.yaml
files:
- playbooks/install-ansible.yaml
- playbooks/roles/backup
- playbooks/zuul/templates/host_vars/backup
- testinfra/test_backups.py
- job:
name: system-config-run-mirror-base
parent: system-config-run
abstract: true
description: |
Run the playbook for a mirror node
vars:
run_playbooks:
- playbooks/service-letsencrypt.yaml
- playbooks/service-mirror.yaml
host-vars:
mirror01.openafs.provider.opendev.org:
host_copy_output:
'/var/log/apache2/': logs
'/var/log/acme.sh': logs
'/etc/apache2/sites-available/mirror.conf': logs
mirror02.openafs.provider.opendev.org:
host_copy_output:
'/var/log/apache2/': logs
'/var/log/acme.sh': logs
'/etc/apache2/sites-available/mirror.conf': logs
files:
- playbooks/install-ansible.yaml
- roles/
- playbooks/group_vars/mirror.yaml
- playbooks/roles/mirror/
- playbooks/roles/letsencrypt
- playbooks/service-letsencrypt.yaml
- playbooks/service-mirror.yaml
- playbooks/zuul/templates/group_vars/mirror.yaml.j2
- testinfra/test_mirror.py
- job:
name: system-config-run-mirror-x86
parent: system-config-run-mirror-base
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: mirror01.openafs.provider.opendev.org
label: ubuntu-bionic
- name: mirror02.openafs.provider.opendev.org
label: ubuntu-focal
- job:
name: system-config-run-mirror-arm64
parent: system-config-run-mirror-base
timeout: 3600
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic-arm64
- name: mirror01.openafs.provider.opendev.org
label: ubuntu-bionic-arm64
- job:
name: system-config-run-mirror-update
parent: system-config-run
description: |
Run the playbook for a mirror update node
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: mirror-update01.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-mirror-update.yaml
files:
- playbooks/install-ansible.yaml
- roles/
- playbooks/roles/mirror-update/
- playbooks/service-mirror-update.yaml
- testinfra/test_mirror-update.py
- job:
name: system-config-run-docker-registry
parent: system-config-run
description: |
Run the playbook for the docker registry.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: insecure-ci-registry01.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-letsencrypt.yaml
- playbooks/service-registry.yaml
host-vars:
insecure-ci-registry01.opendev.org:
host_copy_output:
'/var/registry/auth': logs
'/var/registry/certs': logs
files:
- playbooks/install-ansible.yaml
- playbooks/group_vars/registry.yaml
- playbooks/host_vars/insecure-ci-registry\d+.opendev.org.yaml
- playbooks/zuul/templates/group_vars/registry.yaml.j2
- playbooks/roles/letsencrypt-create-certs/handlers/restart_zuul_registry.yaml
- playbooks/roles/registry/
- playbooks/roles/pip3/
- playbooks/roles/install-docker/
- testinfra/test_registry.py
- job:
name: system-config-run-etherpad
parent: system-config-run-containers
description: |
Run the playbook for the etherpad servers.
timeout: 3600
requires: etherpad-container-image
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: etherpad01.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-letsencrypt.yaml
- playbooks/service-etherpad.yaml
files:
- playbooks/bridge.yaml
- playbooks/service-letsencrypt.yaml
- playbooks/service-etherpad.yaml
- playbooks/roles/etherpad/
- playbooks/roles/install-docker/
- playbooks/roles/pip3/
- docker/etherpad/
- job:
name: system-config-run-gitea
parent: system-config-run-containers
description: |
Run the playbook for the gitea servers.
timeout: 3600
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: gitea-lb01.opendev.org
label: ubuntu-bionic
- name: gitea99.opendev.org
label: ubuntu-bionic
required-projects:
- openstack/project-config
- opendev/system-config
vars:
run_playbooks:
- playbooks/service-letsencrypt.yaml
- playbooks/service-gitea-lb.yaml
- playbooks/service-gitea.yaml
- playbooks/manage-projects.yaml
run_test_playbook: playbooks/test-gitea.yaml
host-vars:
gitea99.opendev.org:
host_copy_output:
'/var/gitea/conf': logs
'/var/gitea/certs': logs
'/var/gitea/logs': logs
gitea-lb01.opendev.org:
host_copy_output:
'/var/haproxy/etc': logs
files:
- playbooks/install-ansible.yaml
- playbooks/service-letsencrypt.yaml
- playbooks/service-gitea-lb.yaml
- playbooks/service-gitea.yaml
- playbooks/manage-projects.yaml
- playbooks/test-gitea.yaml
- playbooks/group_vars/gitea.yaml
- playbooks/group_vars/gitea-lb.yaml
- playbooks/host_vars/gitea
- playbooks/zuul/templates/group_vars/gitea.yaml.j2
- playbooks/zuul/templates/group_vars/gitea-lb.yaml.j2
- playbooks/roles/pip3/
- playbooks/roles/install-docker/
- playbooks/roles/letsencrypt
- playbooks/roles/gerrit/
- playbooks/roles/gitea/
- playbooks/roles/gitea-git-repos/
- playbooks/roles/haproxy/
- playbooks/roles/letsencrypt-create-certs/handlers/restart_gitea.yaml
- testinfra/test_gitea.py
- testinfra/test_gitea_lb.py
# From gitea_files -- If we rebuild the image, we want to run
# this job as well.
- docker/gitea/
# From haproxy-statsd_files -- If we rebuild the image, we want
# to run this job as well.
- docker/haproxy-statsd/
- job:
name: system-config-run-meetpad
parent: system-config-run-containers
requires: jitsi-meet-container-image
description: |
Run the playbook for jitsi-meet.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: meetpad01.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-letsencrypt.yaml
- playbooks/service-meetpad.yaml
host-vars:
meetpad01.opendev.org:
host_copy_output:
'/var/jitsi-meet': logs
files:
- playbooks/install-ansible.yaml
- playbooks/group_vars/meetpad.yaml
- playbooks/host_vars/meetpad\d+.opendev.org.yaml
- playbooks/roles/letsencrypt-create-certs/handlers/restart_jitsi_meet.yaml
- playbooks/roles/jitsi-meet/
- playbooks/roles/pip3/
- playbooks/roles/install-docker/
- playbooks/zuul/templates/group_vars/meetpad.yaml.j2
- testinfra/test_meetpad.py
- docker/jitsi-meet/
- job:
name: system-config-run-zookeeper
parent: system-config-run
description: |
Run the playbook for the zookeeper cluster.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: zk01.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-zookeeper.yaml
files:
- playbooks/install-ansible.yaml
- playbooks/group_vars/zookeeper.yaml
- ^playbooks/host_vars/zk\d+\..*
- playbooks/roles/zookeeper/
- playbooks/roles/pip3/
- playbooks/roles/install-docker/
- testinfra/test_zookeeper.py
- job:
name: system-config-run-zuul-preview
parent: system-config-run
description: |
Run the playbook for the zuul-preview service.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: zp01.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-zuul-preview.yaml
files:
- playbooks/install-ansible.yaml
- playbooks/roles/zuul-preview/
- playbooks/roles/pip3/
- playbooks/roles/install-docker/
- testinfra/test_zuul_preview.py
- job:
name: system-config-run-zuul
parent: system-config-run
description: |
Run the playbook for the main Zuul cluster.
timeout: 3600
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: zk01.opendev.org
label: ubuntu-bionic
- name: zm01.openstack.org
label: ubuntu-xenial
- name: ze01.opendev.org
label: ubuntu-focal
- name: ze01.openstack.org
label: ubuntu-xenial
- name: zuul01.openstack.org
label: ubuntu-xenial
required-projects:
- openstack/project-config
- opendev/system-config
vars:
run_playbooks:
- playbooks/service-letsencrypt.yaml
- playbooks/service-zookeeper.yaml
- playbooks/service-zuul.yaml
host-vars:
zm01.openstack.org:
host_copy_output:
'/etc/zuul/zuul.conf': logs
ze01.opendev.org:
host_copy_output:
'/etc/zuul/zuul.conf': logs
ze01.openstack.org:
host_copy_output:
'/etc/zuul/zuul.conf': logs
zuul01.openstack.org:
host_copy_output:
'/etc/zuul/zuul.conf': logs
files:
- playbooks/install-ansible.yaml
- playbooks/service-zookeeper.yaml
- playbooks/service-zuul.yaml
- playbooks/group_vars/zuul
- playbooks/group_vars/zookeeper.yaml
- playbooks/host_vars/zk\d+
- playbooks/host_vars/zuul01.openstack.org
- playbooks/roles/zookeeper/
- playbooks/roles/zuul
- job:
name: system-config-run-review
parent: system-config-run-containers
description: |
Run the playbook for gerrit (in a container).
requires: gerrit-2.13-container-image
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: review01.openstack.org
label: ubuntu-xenial
- name: review-dev01.opendev.org
label: ubuntu-xenial
required-projects:
- openstack/project-config
- opendev/system-config
vars:
run_playbooks:
- playbooks/service-letsencrypt.yaml
- playbooks/service-review-dev.yaml
- playbooks/service-review.yaml
host-vars:
review-dev01.opendev.org:
host_copy_output:
'/home/gerrit2/review_site/etc': logs
'/home/gerrit2/review_site/logs': logs
'/var/log/apache2/': logs
'/var/log/acme.sh': logs
review01.openstack.org:
host_copy_output:
'/home/gerrit2/review_site/etc': logs
'/home/gerrit2/review_site/logs': logs
'/var/log/apache2/': logs
'/var/log/acme.sh': logs
files:
- playbooks/install-ansible.yaml
- playbooks/service-letsencrypt.yaml
- playbooks/service-review.*.yaml
- playbooks/group_vars/review.yaml
- playbooks/host_vars/review\d+.openstack.org.yaml
- playbooks/group_vars/review-dev.yaml
- playbooks/host_vars/review-dev\d+.opendev.org.yaml
- playbooks/roles/pip3/
- playbooks/roles/install-docker/
- playbooks/roles/letsencrypt
- playbooks/roles/gerrit/
- playbooks/zuul/templates/group_vars/review-dev.yaml.j2
- playbooks/zuul/templates/group_vars/review.yaml.j2
- testinfra/test_gerrit.py
- docker/gerrit/2.13/
- job:
name: system-config-run-static
parent: system-config-run
description: |
Run the playbook for a static node.
timeout: 3600
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: static01.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-letsencrypt.yaml
- playbooks/service-static.yaml
files:
- playbooks/install-ansible.yaml
- playbooks/roles/static/
- playbooks/roles/letsencrypt
- playbooks/service-letsencrypt.yaml
- playbooks/service-static.yaml
- testinfra/test_static.py
host-vars:
static01.opendev.org:
host_copy_output:
'/var/log/acme.sh/': logs
'/etc/apache2/': logs
'/var/log/apache2/': logs