system-config/zuul.d/system-config-run.yaml
Clark Boylan 9b5e5d3c57 Deal with gitea pagination of repo lists
We list gitea repos to determine if we need to create a repo. If the
repo isn't listed by gitea we create it. New gitea paginates these
listings so we were only getting 30 repos listed when we had far more.
This resulted in us trying to create repos which already exist which is
a gitea http 409 error.

Fix this by paging through the listings until we've seen all the
repos. This should give us a complete listing.

To test this we run our manage-projects playbook twice in the
system-config-run-gitea job. The first pass creates all the new
projects. Then the second pass should noop cleanly.

Change-Id: I73b77b9ddaa0106d4dc0a49c4d4b7751a39a16f9
Co-Authored-By: Jeremy Stanley <fungi@yuggoth.org>
2020-06-25 13:51:27 -07:00

782 lines
24 KiB
YAML

- job:
name: system-config-run
description: |
Run the "base" playbook for system-config hosts.
This is a parent job designed to be inherited.
abstract: true
pre-run: playbooks/zuul/run-base-pre.yaml
run: playbooks/zuul/run-base.yaml
post-run: playbooks/zuul/run-base-post.yaml
vars:
install_ansible_ara_enable: true
zuul_copy_output: "{{ copy_output | combine(host_copy_output | default({})) }}"
stage_dir: "{{ ansible_user_dir }}/zuul-output"
copy_output:
'/var/log/syslog': logs_txt
'/var/log/messages': logs_txt
'/var/log/docker': logs
'/etc/iptables/rules.v4': logs_txt
'/etc/iptables/rules.v6': logs_txt
host-vars:
bridge.openstack.org:
host_copy_output:
'{{ zuul.project.src_dir }}/junit.xml': logs
# Note: the following two jobs implement the variant-based multiple
# inheritance trick. Both of these variants will always apply,
# therefore both parents will appear in the inheritance hierarchy).
- job:
name: system-config-run-containers
parent: system-config-run
# Note: see above re multiple-inheritance.
- job:
name: system-config-run-containers
parent: opendev-buildset-registry-consumer
description: |
Run the "base" playbook for system-config hosts which use
containers.
This is a parent job designed to be inherited. Use this job if
the service in question is container-based. It expects a
buildset registry and pulls images from the intermediate
registry.
- job:
name: system-config-run-base
parent: system-config-run
description: |
Run the "base" playbook on each of the node types
currently in use.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: xenial
label: ubuntu-xenial
- name: bionic
label: ubuntu-bionic
- name: focal
label: ubuntu-focal
files:
- playbooks/
- roles/
- testinfra/
- job:
name: system-config-run-base-ansible-devel
parent: system-config-run-base
description: |
Run the base playbook with the latest ansible.
required-projects:
- name: github.com/ansible/ansible
override-checkout: devel
- name: github.com/philpep/testinfra
- name: openstack/openstacksdk
- name: recordsansible/ara
# NOTE(ianw): 2019-06-11 pinned to stable branch until we handle 1.0
# http://lists.openstack.org/pipermail/openstack-infra/2019-June/006400.html
override-checkout: stable/0.x
vars:
bridge_ansible_name: '{{ ansible_user_dir}}/src/github.com/ansible/ansible'
bridge_ansible_version: null
bridge_openstacksdk_name: '{{ ansible_user_dir }}/src/opendev.org/openstack/openstacksdk'
bridge_openstacksdk_version: null
bridge_ara_name: '{{ ansible_user_dir}}/src/opendev.org/recordsansible/ara'
bridge_ara_version: null
# Although we don't have an arm64 based bridge; Zuul can't currently
# allocate a mixed x86/arm64 situation across clouds. Thus it helps
# to keep this clean so we can run the other tests.
- job:
name: system-config-run-base-arm64
parent: system-config-run
description: |
Run the "base" playbook on ARM64.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic-arm64
- name: bionic
label: ubuntu-bionic-arm64
files:
- playbooks/
- roles/
- testinfra/
- job:
name: system-config-run-eavesdrop
parent: system-config-run-containers
description: |
Run the playbook for an eavesdrop server.
required-projects:
- opendev/ansible-role-puppet
- opendev/system-config
- openstack/project-config
requires: accessbot-container-image
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: eavesdrop01.openstack.org
label: ubuntu-xenial
vars:
run_playbooks:
- playbooks/service-eavesdrop.yaml
files:
- playbooks/service-eavesdrop.yaml
- playbooks/run-accessbot.yaml
- inventory/service/group_vars/eavesdrop.yaml
- inventory/service/group_vars/puppet.yaml
- playbooks/roles/zuul-user
- playbooks/roles/install-docker
- playbooks/roles/puppet-install/
- playbooks/roles/disable-puppet-agent/
- playbooks/roles/accessbot
- playbooks/roles/logrotate
- modules/openstack_project/manifests/eavesdrop.pp
- manifests/eavesdrop.pp
- docker/accessbot/
- testinfra/test_eavesdrop.py
- job:
name: system-config-run-codesearch
parent: system-config-run
description: |
Run the playbook for an codesearch server.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: codesearch01.openstack.org
label: ubuntu-xenial
required-projects:
- opendev/ansible-role-puppet
- opendev/system-config
- openstack/project-config
files:
- playbooks/install-ansible.yaml
- playbooks/service-codesearch.yaml
- inventory/service/group_vars/puppet.yaml
- playbooks/roles/run-puppet/
- playbooks/roles/install-ansible-roles/
- playbooks/roles/sync-project-config
- playbooks/roles/puppet-install/
- playbooks/roles/disable-puppet-agent/
- modules/openstack_project/manifests/codesearch.pp
- modules/openstack_project/files/resync-hound-config.sh
- manifests/codesearch.pp
vars:
run_playbooks:
- playbooks/service-codesearch.yaml
- job:
name: system-config-run-letsencrypt
parent: system-config-run
description: |
Run the playbook for letsencrypt key acquisition
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: adns-letsencrypt.opendev.org
label: ubuntu-bionic
- name: letsencrypt01.opendev.org
label: ubuntu-bionic
- name: letsencrypt02.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-nameserver.yaml
- playbooks/letsencrypt.yaml
host-vars:
bridge.openstack.org:
host_copy_output:
'/var/lib/certcheck': logs
letsencrypt01.opendev.org:
host_copy_output:
'/var/log/acme.sh': logs
letsencrypt02.opendev.org:
host_copy_output:
'/var/log/acme.sh': logs
files:
- playbooks/install-ansible.yaml
- inventory/service/group_vars/letsencrypt.yaml
- playbooks/roles/letsencrypt
- job:
name: system-config-run-lists
parent: system-config-run
description: |
Run the playbook for a list server.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: lists.openstack.org
label: ubuntu-xenial
required-projects:
- opendev/ansible-role-puppet
- opendev/system-config
files:
- playbooks/install-ansible.yaml
- modules/openstack_project/manifests/lists.pp
- inventory/service/host_vars/lists.openstack.org.yaml
- inventory/service/group_vars/puppet
- playbooks/roles/run-puppet/
- playbooks/roles/install-ansible-roles/
- playbooks/roles/base/exim
vars:
run_playbooks:
- playbooks/remote_puppet_else.yaml
- job:
name: system-config-run-nodepool
parent: system-config-run
description: |
Run the playbook for nodepool.
timeout: 3600
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: zk01.opendev.org
label: ubuntu-bionic
- name: nl01.openstack.org
label: ubuntu-xenial
- name: nb04.opendev.org
label: ubuntu-bionic
required-projects:
- opendev/ansible-role-puppet
- opendev/system-config
- openstack/project-config
host-vars:
nl01.openstack.org:
host_copy_output:
'/etc/nodepool/nodepool.yaml': logs
'/var/log/nodepool/launcher-debug.log': logs
nb04.opendev.org:
host_copy_output:
'/etc/nodepool/nodepool.yaml': logs
'/var/log/nodepool/builder-debug.log': logs
vars:
run_playbooks:
- playbooks/letsencrypt.yaml
- playbooks/service-zookeeper.yaml
- playbooks/service-nodepool.yaml
- playbooks/remote_puppet_else.yaml
# Test our ad hoc restart playbook works
- playbooks/nodepool_restart.yaml
files:
- playbooks/install-ansible.yaml
- inventory/service/group_vars/nodepool.yaml
- inventory/service/group_vars/nodepool-builder.yaml
- inventory/service/group_vars/nodepool-launcher.yaml
- inventory/service/group_vars/puppet
- playbooks/roles/run-puppet/
- playbooks/roles/install-ansible-roles/
- playbooks/roles/configure-openstacksdk/
- playbooks/roles/nodepool
- playbooks/templates/clouds/
- playbooks/nodepool_restart.yaml
- testinfra/test_nodepool.py
- job:
name: system-config-run-dns
parent: system-config-run
description: |
Run the playbook for dns.
required-projects:
- opendev/zone-opendev.org
- opendev/zone-zuul-ci.org
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: adns1.opendev.org
label: ubuntu-bionic
- name: ns1.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-nameserver.yaml
host-vars:
adns1.opendev.org:
host_copy_output:
'/etc/bind/named.conf': logs
'/var/lib/bind/zones': logs
files:
- playbooks/install-ansible.yaml
- inventory/service/group_vars/adns.yaml
- inventory/service/group_vars/dns.yaml
- inventory/service/host_vars/(ad)?ns\d+.opendev.org.yaml
- playbooks/zuul/templates/group_vars/adns.yaml.j2
- playbooks/zuul/templates/group_vars/ns.yaml.j2
- playbooks/roles/master-nameserver/
- playbooks/roles/nameserver/
- testinfra/test_adns.py
- testinfra/test_ns.py
- job:
name: system-config-run-backup
parent: system-config-run
description: |
Run the playbook for backup configuration
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: backup01.region.provider.opendev.org
label: ubuntu-bionic
- name: backup-test01.opendev.org
label: ubuntu-bionic
- name: backup-test02.opendev.org
label: ubuntu-xenial
vars:
run_playbooks:
- playbooks/service-backup.yaml
files:
- playbooks/install-ansible.yaml
- playbooks/roles/backup
- playbooks/zuul/templates/host_vars/backup
- testinfra/test_backups.py
- job:
name: system-config-run-mirror-base
parent: system-config-run
abstract: true
description: |
Run the playbook for a mirror node
vars:
run_playbooks:
- playbooks/letsencrypt.yaml
- playbooks/service-mirror.yaml
host-vars:
mirror01.openafs.provider.opendev.org:
host_copy_output:
'/var/log/apache2/': logs
'/var/log/acme.sh': logs
'/etc/apache2/sites-available/mirror.conf': logs
mirror02.openafs.provider.opendev.org:
host_copy_output:
'/var/log/apache2/': logs
'/var/log/acme.sh': logs
'/etc/apache2/sites-available/mirror.conf': logs
files:
- playbooks/install-ansible.yaml
- roles/
- inventory/service/group_vars/mirror.yaml
- playbooks/roles/mirror/
- playbooks/roles/letsencrypt
- playbooks/letsencrypt.yaml
- playbooks/service-mirror.yaml
- playbooks/zuul/templates/group_vars/mirror.yaml.j2
- testinfra/test_mirror.py
- job:
name: system-config-run-mirror-x86
parent: system-config-run-mirror-base
timeout: 3600
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: mirror01.openafs.provider.opendev.org
label: ubuntu-bionic
- name: mirror02.openafs.provider.opendev.org
label: ubuntu-focal
- job:
name: system-config-run-mirror-arm64
parent: system-config-run-mirror-base
timeout: 3600
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic-arm64
- name: mirror01.openafs.provider.opendev.org
label: ubuntu-bionic-arm64
- job:
name: system-config-run-mirror-update
parent: system-config-run
description: |
Run the playbook for a mirror update node
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: mirror-update01.opendev.org
label: ubuntu-focal
vars:
run_playbooks:
- playbooks/service-mirror-update.yaml
files:
- playbooks/install-ansible.yaml
- roles/
- playbooks/roles/mirror-update/
- playbooks/service-mirror-update.yaml
- testinfra/test_mirror-update.py
- job:
name: system-config-run-docker-registry
parent: system-config-run
description: |
Run the playbook for the docker registry.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: insecure-ci-registry01.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/letsencrypt.yaml
- playbooks/service-registry.yaml
host-vars:
insecure-ci-registry01.opendev.org:
host_copy_output:
'/var/registry/auth': logs
'/var/registry/certs': logs
files:
- playbooks/install-ansible.yaml
- inventory/service/group_vars/registry.yaml
- inventory/service/host_vars/insecure-ci-registry\d+.opendev.org.yaml
- playbooks/zuul/templates/group_vars/registry.yaml.j2
- playbooks/roles/letsencrypt-create-certs/handlers/restart_zuul_registry.yaml
- playbooks/roles/registry/
- playbooks/roles/pip3/
- playbooks/roles/install-docker/
- testinfra/test_registry.py
- job:
name: system-config-run-etherpad
parent: system-config-run-containers
description: |
Run the playbook for the etherpad servers.
timeout: 3600
requires: etherpad-container-image
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: etherpad01.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/letsencrypt.yaml
- playbooks/service-etherpad.yaml
files:
- playbooks/bridge.yaml
- playbooks/letsencrypt.yaml
- playbooks/service-etherpad.yaml
- playbooks/roles/etherpad/
- playbooks/roles/install-docker/
- playbooks/roles/pip3/
- docker/etherpad/
- testinfra/test_etherpad.py
- job:
name: system-config-run-gitea
parent: system-config-run-containers
description: |
Run the playbook for the gitea servers.
timeout: 3600
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: gitea-lb01.opendev.org
label: ubuntu-bionic
- name: gitea99.opendev.org
label: ubuntu-bionic
required-projects:
- openstack/project-config
- opendev/system-config
vars:
run_playbooks:
- playbooks/letsencrypt.yaml
- playbooks/service-gitea-lb.yaml
- playbooks/service-gitea.yaml
- playbooks/manage-projects.yaml
# Run twice to ensure that we noop properly when
# all projects are created in gitea.
- playbooks/manage-projects.yaml
run_test_playbook: playbooks/test-gitea.yaml
host-vars:
gitea99.opendev.org:
host_copy_output:
'/var/gitea/conf': logs
'/var/gitea/certs': logs
'/var/gitea/logs': logs
gitea-lb01.opendev.org:
host_copy_output:
'/var/haproxy/etc': logs
files:
- playbooks/install-ansible.yaml
- playbooks/letsencrypt.yaml
- playbooks/service-gitea-lb.yaml
- playbooks/service-gitea.yaml
- playbooks/manage-projects.yaml
- playbooks/test-gitea.yaml
- inventory/service/group_vars/gitea.yaml
- inventory/service/group_vars/gitea-lb.yaml
- inventory/service/host_vars/gitea
- playbooks/zuul/templates/group_vars/gitea.yaml.j2
- playbooks/zuul/templates/group_vars/gitea-lb.yaml.j2
- playbooks/roles/pip3/
- playbooks/roles/install-docker/
- playbooks/roles/letsencrypt
- playbooks/roles/gerrit/
- playbooks/roles/gitea/
- playbooks/roles/gitea-git-repos/
- playbooks/roles/haproxy/
- playbooks/roles/letsencrypt-create-certs/handlers/restart_gitea.yaml
- testinfra/test_gitea.py
- testinfra/test_gitea_lb.py
# From gitea_files -- If we rebuild the image, we want to run
# this job as well.
- docker/gitea/
# From haproxy-statsd_files -- If we rebuild the image, we want
# to run this job as well.
- docker/haproxy-statsd/
- job:
name: system-config-run-meetpad
parent: system-config-run-containers
requires: jitsi-meet-container-image
description: |
Run the playbook for jitsi-meet.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: meetpad01.opendev.org
label: ubuntu-bionic
- name: jvb01.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/letsencrypt.yaml
- playbooks/service-meetpad.yaml
host-vars:
meetpad01.opendev.org:
host_copy_output:
'/var/jitsi-meet': logs
jvb01.opendev.org:
host_copy_output:
'/var/jitsi-meet': logs
files:
- playbooks/install-ansible.yaml
- inventory/service/group_vars/meetpad.yaml
- inventory/service/host_vars/meetpad\d+.opendev.org.yaml
- playbooks/roles/letsencrypt-create-certs/handlers/restart_jitsi_meet.yaml
- playbooks/roles/jitsi-meet/
- playbooks/roles/pip3/
- playbooks/roles/install-docker/
- playbooks/zuul/templates/group_vars/meetpad.yaml.j2
- testinfra/test_meetpad.py
- docker/jitsi-meet/
- job:
name: system-config-run-zookeeper
parent: system-config-run
description: |
Run the playbook for the zookeeper cluster.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: zk01.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-zookeeper.yaml
files:
- playbooks/install-ansible.yaml
- inventory/service/group_vars/zookeeper.yaml
- ^inventory/service/host_vars/zk\d+\..*
- playbooks/roles/zookeeper/
- playbooks/roles/pip3/
- playbooks/roles/install-docker/
- testinfra/test_zookeeper.py
- job:
name: system-config-run-zuul-preview
parent: system-config-run
description: |
Run the playbook for the zuul-preview service.
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: zp01.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/service-zuul-preview.yaml
files:
- playbooks/install-ansible.yaml
- playbooks/roles/zuul-preview/
- playbooks/roles/pip3/
- playbooks/roles/install-docker/
- testinfra/test_zuul_preview.py
- job:
name: system-config-run-zuul
parent: system-config-run
description: |
Run the playbook for the main Zuul cluster.
timeout: 3600
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: zk01.opendev.org
label: ubuntu-bionic
- name: zm01.openstack.org
label: ubuntu-xenial
- name: ze01.opendev.org
label: ubuntu-focal
- name: ze01.openstack.org
label: ubuntu-xenial
- name: zuul01.openstack.org
label: ubuntu-xenial
required-projects:
- openstack/project-config
- opendev/system-config
vars:
run_playbooks:
- playbooks/letsencrypt.yaml
- playbooks/service-zookeeper.yaml
- playbooks/service-zuul.yaml
# Test our ad hoc restart playbook works
- playbooks/zuul_restart.yaml
host-vars:
zm01.openstack.org:
host_copy_output:
'/etc/hosts': logs
'/etc/zuul/zuul.conf': logs
'/var/log/zuul/merger-debug.log': logs
ze01.opendev.org:
host_copy_output:
'/etc/hosts': logs
'/etc/zuul/zuul.conf': logs
'/var/log/zuul/executor-debug.log': logs
ze01.openstack.org:
host_copy_output:
'/etc/hosts': logs
'/etc/zuul/zuul.conf': logs
'/var/log/zuul/executor-debug.log': logs
zuul01.openstack.org:
host_copy_output:
'/etc/hosts': logs
'/etc/zuul/zuul.conf': logs
'/var/log/zuul/debug.log': logs
bridge.openstack.org:
host_copy_output:
'/etc/hosts': logs
files:
- playbooks/install-ansible.yaml
- playbooks/service-zookeeper.yaml
- playbooks/service-zuul.yaml
- inventory/service/group_vars/zuul
- inventory/service/group_vars/zookeeper.yaml
- inventory/service/host_vars/zk\d+
- inventory/service/host_vars/zuul01.openstack.org
- playbooks/roles/zookeeper/
- playbooks/roles/install-apt-repo
- playbooks/roles/zuul
- playbooks/zuul/templates/group_vars/zuul
- playbooks/zuul/templates/group_vars/zookeeper.yaml
- playbooks/zuul/templates/host_vars/zk\d+
- playbooks/zuul/templates/host_vars/zuul01.openstack.org
- playbooks/zuul_restart.yaml
- testinfra/test_zuul_executor.py
- testinfra/test_zuul_scheduler.py
- testinfra/test_zuul_merger.py
- testinfra/util.py
- job:
name: system-config-run-review
parent: system-config-run-containers
description: |
Run the playbook for gerrit (in a container).
requires: gerrit-2.13-container-image
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: review01.openstack.org
label: ubuntu-xenial
- name: review-dev01.opendev.org
label: ubuntu-xenial
required-projects:
- openstack/project-config
- opendev/system-config
vars:
run_playbooks:
- playbooks/letsencrypt.yaml
- playbooks/service-review-dev.yaml
- playbooks/service-review.yaml
host-vars:
review-dev01.opendev.org:
host_copy_output:
'/home/gerrit2/review_site/etc': logs
'/home/gerrit2/review_site/logs': logs
'/var/log/apache2/': logs
'/var/log/acme.sh': logs
review01.openstack.org:
host_copy_output:
'/home/gerrit2/review_site/etc': logs
'/home/gerrit2/review_site/logs': logs
'/var/log/apache2/': logs
'/var/log/acme.sh': logs
files:
- playbooks/install-ansible.yaml
- playbooks/letsencrypt.yaml
- playbooks/service-review.*.yaml
- inventory/service/group_vars/review.yaml
- inventory/service/host_vars/review\d+.openstack.org.yaml
- inventory/service/group_vars/review-dev.yaml
- inventory/service/host_vars/review-dev\d+.opendev.org.yaml
- playbooks/roles/pip3/
- playbooks/roles/install-docker/
- playbooks/roles/letsencrypt
- playbooks/roles/gerrit/
- playbooks/zuul/templates/group_vars/review-dev.yaml.j2
- playbooks/zuul/templates/group_vars/review.yaml.j2
- testinfra/test_gerrit.py
- docker/gerrit/2.13/
- job:
name: system-config-run-static
parent: system-config-run
description: |
Run the playbook for a static node.
timeout: 3600
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: static01.opendev.org
label: ubuntu-bionic
vars:
run_playbooks:
- playbooks/letsencrypt.yaml
- playbooks/service-static.yaml
files:
- playbooks/install-ansible.yaml
- playbooks/roles/static/
- playbooks/roles/letsencrypt
- playbooks/letsencrypt.yaml
- playbooks/service-static.yaml
- testinfra/test_static.py
host-vars:
static01.opendev.org:
host_copy_output:
'/var/log/acme.sh/': logs
'/etc/apache2/': logs
'/var/log/apache2/': logs