From 533594d9592ccd9ca4482f462d68f7ab34b2d6e6 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Mon, 10 May 2021 08:52:32 -0700 Subject: [PATCH] Add zuul02 to inventory This zuul02 instance will replace zuul01. There are a few items to coordinate when doing an actual switch so we haven't removed zuul01 from inventory here. In particular we need to update gearman server config values in the zuul cluster and we need to save queues, shutdown zuul01, then start zuul02's scheduler and restore queues there. I believe landing this change is safe as we don't appear to start zuul on new instances by default. Reviewers should double check this. Depends-On: https://review.opendev.org/c/opendev/zone-opendev.org/+/791039 Change-Id: I524b456e494124d8293fbe8e1468de40f3800772 --- hiera/common.yaml | 1 + inventory/base/hosts.yaml | 7 +++++++ inventory/service/group_vars/zuul.yaml | 2 ++ inventory/service/host_vars/zuul02.opendev.org.yaml | 10 ++++++++++ .../roles/letsencrypt-create-certs/handlers/main.yaml | 3 +++ testinfra/test_zuul_scheduler.py | 2 +- zuul.d/infra-prod.yaml | 1 + zuul.d/system-config-run.yaml | 10 +++++----- 8 files changed, 30 insertions(+), 6 deletions(-) create mode 100644 inventory/service/host_vars/zuul02.opendev.org.yaml diff --git a/hiera/common.yaml b/hiera/common.yaml index 79e774eb3a..881a2a1331 100644 --- a/hiera/common.yaml +++ b/hiera/common.yaml @@ -323,6 +323,7 @@ cacti_hosts: - zm07.opendev.org - zm08.opendev.org - zuul01.openstack.org +- zuul02.opendev.org statusbot_auth_nicks: - jeblair - corvus diff --git a/inventory/base/hosts.yaml b/inventory/base/hosts.yaml index 8a34f3abc5..cf65a3e706 100644 --- a/inventory/base/hosts.yaml +++ b/inventory/base/hosts.yaml @@ -892,6 +892,13 @@ all: region_name: DFW public_v4: 104.130.246.196 public_v6: 2001:4800:7819:103:be76:4eff:fe04:5870 + zuul02.opendev.org: + ansible_host: 104.130.246.31 + location: + cloud: openstackci-rax + region_name: DFW + public_v4: 104.130.246.31 + public_v6: 2001:4800:7819:103:be76:4eff:fe04:42c2 zp01.opendev.org: ansible_host: 174.143.130.226 location: diff --git a/inventory/service/group_vars/zuul.yaml b/inventory/service/group_vars/zuul.yaml index 7bc0b6e680..6e5ea74992 100644 --- a/inventory/service/group_vars/zuul.yaml +++ b/inventory/service/group_vars/zuul.yaml @@ -6,6 +6,8 @@ zuul_known_hosts: | [review.opendev.org]:29418,[review.openstack.org]:29418,[104.130.246.32]:29418,[2001:4800:7819:103:be76:4eff:fe04:9229]:29418 {{ gerrit_ssh_rsa_pubkey_contents }} [git.opendaylight.org]:29418,[52.35.122.251]:29418,[2600:1f14:421:f500:7b21:2a58:ab0a:2d17]:29418 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAyRXyHEw/P1iZr/fFFzbodT5orVV/ftnNRW59Zh9rnSY5Rmbc9aygsZHdtiWBERVVv8atrJSdZool75AglPDDYtPICUGWLR91YBSDcZwReh5S9es1dlQ6fyWTnv9QggSZ98KTQEuE3t/b5SfH0T6tXWmrNydv4J2/mejKRRLU2+oumbeVN1yB+8Uau/3w9/K5F5LgsDDzLkW35djLhPV8r0OfmxV/cAnLl7AaZlaqcJMA+2rGKqM3m3Yu+pQw4pxOfCSpejlAwL6c8tA9naOvBkuJk+hYpg5tDEq2QFGRX5y1F9xQpwpdzZROc5hdGYntM79VMMXTj+95dwVv/8yTsw== gearman_server: zuul01.openstack.org +# TODO this swap needs to happen as part of the server swap +#gearman_server: zuul02.opendev.org zuul_connections: - name: 'smtp' driver: 'smtp' diff --git a/inventory/service/host_vars/zuul02.opendev.org.yaml b/inventory/service/host_vars/zuul02.opendev.org.yaml new file mode 100644 index 0000000000..0f870bbf11 --- /dev/null +++ b/inventory/service/host_vars/zuul02.opendev.org.yaml @@ -0,0 +1,10 @@ +gearman_server: 127.0.0.1 +letsencrypt_certs: + zuul02-opendev-main: + - zuul.opendev.org + - zuul.openstack.org + - zuul02.opendev.org +borg_backup_excludes_extra: + - /var/lib/zuul/backup/* + - /var/lib/zuul/times/* + - /var/log/zuul/*debug* diff --git a/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml b/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml index 079b51932d..b49a0a6be3 100644 --- a/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml +++ b/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml @@ -26,6 +26,9 @@ - name: letsencrypt updated zuul-opendev-main include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml +- name: letsencrypt updated zuul02-opendev-main + include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml + - name: letsencrypt updated logs-main include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml diff --git a/testinfra/test_zuul_scheduler.py b/testinfra/test_zuul_scheduler.py index ae1ed676bb..e2633a2c3d 100644 --- a/testinfra/test_zuul_scheduler.py +++ b/testinfra/test_zuul_scheduler.py @@ -15,7 +15,7 @@ import util import socket -testinfra_hosts = ['zuul01.openstack.org'] +testinfra_hosts = ['zuul02.opendev.org'] def test_iptables(host): diff --git a/zuul.d/infra-prod.yaml b/zuul.d/infra-prod.yaml index 3b681dd329..7709d110b0 100644 --- a/zuul.d/infra-prod.yaml +++ b/zuul.d/infra-prod.yaml @@ -357,6 +357,7 @@ - inventory/service/group_vars/zookeeper.yaml - inventory/service/host_vars/zk\d+ - inventory/service/host_vars/zuul01.openstack.org + - inventory/service/host_vars/zuul02.opendev.org - playbooks/roles/install-docker/ - playbooks/roles/iptables/ - playbooks/roles/zookeeper/ diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml index 572e00efba..7f7b390be6 100644 --- a/zuul.d/system-config-run.yaml +++ b/zuul.d/system-config-run.yaml @@ -746,8 +746,8 @@ label: ubuntu-focal - name: ze01.opendev.org label: ubuntu-focal - - name: zuul01.openstack.org - label: ubuntu-xenial + - name: zuul02.opendev.org + label: ubuntu-focal required-projects: - openstack/project-config - opendev/system-config @@ -769,7 +769,7 @@ '/etc/hosts': logs '/etc/zuul/zuul.conf': logs '/var/log/zuul/executor-debug.log': logs - zuul01.openstack.org: + zuul02.opendev.org: host_copy_output: '/etc/hosts': logs '/etc/zuul/zuul.conf': logs @@ -784,14 +784,14 @@ - inventory/service/group_vars/zuul - inventory/service/group_vars/zookeeper.yaml - inventory/service/host_vars/zk\d+ - - inventory/service/host_vars/zuul01.openstack.org + - inventory/service/host_vars/zuul02.opendev.org - playbooks/roles/zookeeper/ - playbooks/roles/install-apt-repo - playbooks/roles/zuul - playbooks/zuul/templates/group_vars/zuul - playbooks/zuul/templates/group_vars/zookeeper.yaml - playbooks/zuul/templates/host_vars/zk\d+ - - playbooks/zuul/templates/host_vars/zuul01.openstack.org + - playbooks/zuul/templates/host_vars/zuul02.opendev.org - playbooks/zuul_restart.yaml - testinfra/test_zuul_executor.py - testinfra/test_zuul_scheduler.py