From 0c90c128d733d0754dfe35d1098f7bcbdec5d435 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Fri, 28 Oct 2022 12:16:29 +1100 Subject: [PATCH] Reference bastion through prod_bastion group In thinking harder about the bootstrap process, it struck me that the "bastion" group we have is two separate ideas that become a bit confusing because they share a name. We have the testing and production paths that need to find a single bridge node so they can run their nested Ansible. We've recently merged changes to the setup playbooks to not hard-code the bridge node and they now use groups["bastion"][0] to find the bastion host -- but this group is actually orthogonal to the group of the same name defined in inventory/service/groups.yaml. The testing and production paths are running on the executor, and, as mentioned, need to know the bridge node to log into. For the testing path this is happening via the group created in the job definition from zuul.d/system-config-run.yaml. For the production jobs, this group is populated via the add-bastion-host role which dynamically adds the bridge host and group. Only the *nested* Ansible running on the bastion host reads s-c:inventory/service/groups.yaml. None of the nested-ansible playbooks need to target only the currently active bastion host. For example, we can define as many bridge nodes as we like in the inventory and run service-bridge.yaml against them. It won't matter because the production jobs know the host that is the currently active bridge as described above. So, instead of using the same group name in two contexts, rename the testing/production group "prod_bastion". groups["prod_bastion"][0] will be the host that the testing/production jobs use as the bastion host -- references are updated in this change (i.e. the two places this group is defined -- the group name in the system-config-run jobs, and add-bastion-host for production). We then can return the "bastion" group match to bridge*.opendev.org in inventory/service/groups.yaml. This fixes a bootstrapping problem -- if you launch, say, bridge03.opendev.org the launch node script will now apply the base.yaml playbook against it, and correctly apply all variables from the "bastion" group which now matches this new host. This is what we want to ensure, e.g. the zuul user and keys are correctly populated. The other thing we can do here is change the testing path "prod_bastion" hostname to "bridge99.opendev.org". By doing this we ensure we're not hard-coding for the production bridge host in any way (since if both testing and production are called bridge01.opendev.org we can hide problems). This is a big advantage when we want to rotate the production bridge host, as we can be certain there's no hidden dependencies. Change-Id: I137ab824b9a09ccb067b8d5f0bb2896192291883 --- inventory/service/groups.yaml | 4 +--- playbooks/bootstrap-bridge.yaml | 2 +- .../zuul/roles/add-bastion-host/tasks/main.yaml | 2 +- playbooks/zuul/run-base-post.yaml | 4 ++-- playbooks/zuul/run-base.yaml | 14 +++++++------- ...un-production-bootstrap-bridge-add-rootkey.yaml | 2 +- playbooks/zuul/run-production-playbook-post.yaml | 2 +- playbooks/zuul/run-production-playbook.yaml | 2 +- zuul.d/system-config-run.yaml | 10 +++++----- 9 files changed, 20 insertions(+), 22 deletions(-) diff --git a/inventory/service/groups.yaml b/inventory/service/groups.yaml index 0afce37875..d8fa1c9945 100644 --- a/inventory/service/groups.yaml +++ b/inventory/service/groups.yaml @@ -15,10 +15,8 @@ groups: - afsdb*.open*.org - afs[0-9]*.open*.org - static[0-9]*.opendev.org - # bastion group should should only have one entry because we assume - # groups['bastion'][0] is the bastion host name in several places. bastion: - - bridge01.opendev.org + - bridge*.opendev.org borg-backup: - etherpad[0-9]*.opendev.org - gitea01.opendev.org diff --git a/playbooks/bootstrap-bridge.yaml b/playbooks/bootstrap-bridge.yaml index 794656350c..8191e2ad9c 100644 --- a/playbooks/bootstrap-bridge.yaml +++ b/playbooks/bootstrap-bridge.yaml @@ -12,7 +12,7 @@ # In both cases, the "bastion" group has one entry, which is the # bastion host to run against. -- hosts: bastion[0]:!disabled +- hosts: prod_bastion[0]:!disabled name: "Bridge: bootstrap the bastion host" become: true tasks: diff --git a/playbooks/zuul/roles/add-bastion-host/tasks/main.yaml b/playbooks/zuul/roles/add-bastion-host/tasks/main.yaml index 0f3dcd4d84..7b05c31571 100644 --- a/playbooks/zuul/roles/add-bastion-host/tasks/main.yaml +++ b/playbooks/zuul/roles/add-bastion-host/tasks/main.yaml @@ -1,7 +1,7 @@ - name: Add bastion host to inventory for production playbook add_host: name: 'bridge01.opendev.org' - groups: 'bastion' + groups: 'prod_bastion' ansible_python_interpreter: python3 ansible_user: zuul # Without setting ansible_host directly, mirror-workspace-git-repos diff --git a/playbooks/zuul/run-base-post.yaml b/playbooks/zuul/run-base-post.yaml index 49faeeb82a..da23b7ab92 100644 --- a/playbooks/zuul/run-base-post.yaml +++ b/playbooks/zuul/run-base-post.yaml @@ -28,7 +28,7 @@ - include_role: name: stage-output -- hosts: bastion +- hosts: prod_bastion[0] tasks: - name: Set log directory set_fact: @@ -69,7 +69,7 @@ zuul: artifacts: - name: ARA report - url: '{{ groups["bastion"][0] }}/ara-report/' + url: '{{ groups["prod_bastion"][0] }}/ara-report/' - name: Collect ansible configuration synchronize: diff --git a/playbooks/zuul/run-base.yaml b/playbooks/zuul/run-base.yaml index 2b6a15ed39..c8ef4e680d 100644 --- a/playbooks/zuul/run-base.yaml +++ b/playbooks/zuul/run-base.yaml @@ -5,7 +5,7 @@ cloud_launcher_disable_job: true # setup opendev CA -- hosts: bastion[0] +- hosts: prod_bastion[0] become: true tasks: - name: Make temporary dir for CA generation @@ -57,9 +57,9 @@ creates: '{{ item.file }}' loop: - file: '/etc/opendev-ca/ca.key' - content: '{{ hostvars[groups["bastion"][0]]["_opendev_ca_key"]["content"] }}' + content: '{{ hostvars[groups["prod_bastion"][0]]["_opendev_ca_key"]["content"] }}' - file: '/etc/opendev-ca/ca.crt' - content: '{{ hostvars[groups["bastion"][0]]["_opendev_ca_certificate"]["content"] }}' + content: '{{ hostvars[groups["prod_bastion"][0]]["_opendev_ca_certificate"]["content"] }}' - name: Install and trust certificate shell: @@ -67,7 +67,7 @@ cp /etc/opendev-ca/ca.crt /usr/local/share/ca-certificates/opendev-infra-ca.crt update-ca-certificates -- hosts: bastion[0] +- hosts: prod_bastion[0] become: true tasks: - name: Write inventory on bridge @@ -185,7 +185,7 @@ name: encrypt-logs vars: encrypt_logs_files: '{{ _run_playbooks_logs.files | map(attribute="path") | list }}' - encrypt_logs_artifact_path: '{{ groups["bastion"][0] }}/ansible' + encrypt_logs_artifact_path: '{{ groups["prod_bastion"][0] }}/ansible' encrypt_logs_download_script_path: '/var/log/ansible' - name: Run test playbook @@ -216,7 +216,7 @@ zuul: artifacts: - name: Screenshots - url: '{{ groups["bastion"][0] }}/screenshots' + url: '{{ groups["prod_bastion"][0] }}/screenshots' - name: Allow PBR's git calls to operate in system-config, despite not owning it command: git config --global safe.directory /home/zuul/src/opendev.org/opendev/system-config @@ -240,4 +240,4 @@ zuul: artifacts: - name: testinfra results - url: '{{ groups["bastion"][0] }}/test-results.html' + url: '{{ groups["prod_bastion"][0] }}/test-results.html' diff --git a/playbooks/zuul/run-production-bootstrap-bridge-add-rootkey.yaml b/playbooks/zuul/run-production-bootstrap-bridge-add-rootkey.yaml index ab44d63ab9..ece94af83b 100644 --- a/playbooks/zuul/run-production-bootstrap-bridge-add-rootkey.yaml +++ b/playbooks/zuul/run-production-bootstrap-bridge-add-rootkey.yaml @@ -1,4 +1,4 @@ -- hosts: bastion[0] +- hosts: prod_bastion[0] connection: local tasks: - name: Install root keys diff --git a/playbooks/zuul/run-production-playbook-post.yaml b/playbooks/zuul/run-production-playbook-post.yaml index 30c67379c7..dbf0fa1d5f 100644 --- a/playbooks/zuul/run-production-playbook-post.yaml +++ b/playbooks/zuul/run-production-playbook-post.yaml @@ -2,7 +2,7 @@ roles: - add-bastion-host -- hosts: bastion[0] +- hosts: prod_bastion[0] tasks: - name: Encrypt log when: infra_prod_playbook_encrypt_log|default(False) diff --git a/playbooks/zuul/run-production-playbook.yaml b/playbooks/zuul/run-production-playbook.yaml index 51c3070e27..b139c083f3 100644 --- a/playbooks/zuul/run-production-playbook.yaml +++ b/playbooks/zuul/run-production-playbook.yaml @@ -2,7 +2,7 @@ roles: - add-bastion-host -- hosts: bastion[0] +- hosts: prod_bastion[0] tasks: - name: Run the production playbook and capture logs block: diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml index 18b18f4849..9b79f7803e 100644 --- a/zuul.d/system-config-run.yaml +++ b/zuul.d/system-config-run.yaml @@ -59,7 +59,7 @@ currently in use. nodeset: nodes: - - &bridge_node_x86 { name: bridge01.opendev.org, label: ubuntu-jammy } + - &bridge_node_x86 { name: bridge99.opendev.org, label: ubuntu-jammy } - name: xenial label: ubuntu-xenial - name: bionic @@ -81,7 +81,7 @@ # bridge node. This node will then run a nested Ansible to # test the production playbooks -- *that* Ansible has a # "bastion" group too - - &bastion_group { name: bastion, nodes: [ bridge01.opendev.org ] } + - &bastion_group { name: prod_bastion, nodes: [ bridge99.opendev.org ] } files: - tox.ini - playbooks/ @@ -138,7 +138,7 @@ Run the "base" playbook on ARM64. nodeset: nodes: - - &bridge_node_arm64 { name: bridge01.opendev.org, label: ubuntu-jammy-arm64 } + - &bridge_node_arm64 { name: bridge99.opendev.org, label: ubuntu-jammy-arm64 } - name: bionic label: ubuntu-bionic-arm64 - name: focal @@ -228,7 +228,7 @@ # Make sure this test runs acme.sh letsencrypt_self_generate_tokens: False host-vars: - bridge01.opendev.org: + bridge99.opendev.org: host_copy_output: '/var/lib/certcheck': logs letsencrypt01.opendev.org: @@ -967,7 +967,7 @@ '/etc/hosts': logs '/etc/zuul/zuul.conf': logs '/var/log/zuul/debug.log': logs - bridge01.opendev.org: + bridge99.opendev.org: host_copy_output: '/etc/hosts': logs zuul-lb01.opendev.org: