From ea9e4883cc373770a3644dc830b05aef170f4f0b Mon Sep 17 00:00:00 2001 From: Jonathan Rosser Date: Mon, 28 Sep 2020 13:59:42 +0100 Subject: [PATCH] Use nodepool epel mirror in CI for systemd-networkd package This is failing often in CI and we should use the mirror when possible. The normal variable centos_epel_mirror is not defined in an openstack-ansible user variable until after the AIO host prep has happened, so we must detect the nodepool mirror directly when doing the initial host preparation. This patch refactors gathering the nodepool vars completely, to read them into a new ansible var nodepool_vars where they can be accessed in the whole bootstrap_host role. Depends-On: https://review.opendev.org/755065 Change-Id: Ida3762ffff2140b3e5b287443004fa1f16addd53 --- .../tasks/gather_nodepool_vars.yml | 35 +++++++++++ tests/roles/bootstrap-host/tasks/main.yml | 3 + .../tasks/prepare_aio_config.yml | 59 ++++--------------- .../tasks/prepare_networking.yml | 1 + .../templates/user_variables.aio.yml.j2 | 4 +- 5 files changed, 53 insertions(+), 49 deletions(-) create mode 100644 tests/roles/bootstrap-host/tasks/gather_nodepool_vars.yml diff --git a/tests/roles/bootstrap-host/tasks/gather_nodepool_vars.yml b/tests/roles/bootstrap-host/tasks/gather_nodepool_vars.yml new file mode 100644 index 0000000000..d6d9017ec5 --- /dev/null +++ b/tests/roles/bootstrap-host/tasks/gather_nodepool_vars.yml @@ -0,0 +1,35 @@ +--- +# Copyright 2020, BBC R&D +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Detect whether the host is an OpenStack-CI host + stat: + path: /etc/nodepool + register: nodepool_dir + +- name: Set facts when inside of OpenStack-Infra + when: + - nodepool_dir.stat.exists + block: + - name: Read nodepool environment variables + shell: | + source /etc/ci/mirror_info.sh + env | grep NODEPOOL | sed 's/=/: /g' + args: + executable: /bin/bash + register: nodepool_env + + - name: Set nodepool environment variables fact + set_fact: + nodepool_vars: "{{ nodepool_env.stdout | from_yaml }}" diff --git a/tests/roles/bootstrap-host/tasks/main.yml b/tests/roles/bootstrap-host/tasks/main.yml index 40bf0d6a27..e5d5a2bf82 100644 --- a/tests/roles/bootstrap-host/tasks/main.yml +++ b/tests/roles/bootstrap-host/tasks/main.yml @@ -37,6 +37,9 @@ tags: - always +- name: Gather nodepool variables + include: gather_nodepool_vars.yml + - name: Create the required directories file: path: "{{ item }}" diff --git a/tests/roles/bootstrap-host/tasks/prepare_aio_config.yml b/tests/roles/bootstrap-host/tasks/prepare_aio_config.yml index ab9c8eb16c..3d20555021 100644 --- a/tests/roles/bootstrap-host/tasks/prepare_aio_config.yml +++ b/tests/roles/bootstrap-host/tasks/prepare_aio_config.yml @@ -69,56 +69,21 @@ tags: - generate_secrets -- name: Detect whether the host is an OpenStack-CI host - stat: - path: /etc/nodepool - register: nodepool_dir - - name: Set facts when inside of OpenStack-Infra when: - - nodepool_dir.stat.exists + - nodepool_vars is defined block: - - name: Discover the OpenStack-Infra mirrors - shell: | - source /etc/ci/mirror_info.sh - NODEPOOL_OVERRIDES="/etc/openstack_deploy/user_openstackci.yml" - echo "uca_apt_repo_url: '${NODEPOOL_UCA_MIRROR}'" >> ${NODEPOOL_OVERRIDES} - echo "openstack_hosts_centos_mirror_url: '${NODEPOOL_CENTOS_MIRROR}'" >> ${NODEPOOL_OVERRIDES} - echo "opensuse_mirror: '${NODEPOOL_OPENSUSE_MIRROR}'" >> ${NODEPOOL_OVERRIDES} - echo "centos_epel_mirror: '${NODEPOOL_EPEL_MIRROR}'" >> ${NODEPOOL_OVERRIDES} - echo "galera_percona_xtrabackup_repo_host: '${NODEPOOL_PERCONA_PROXY}'" >> ${NODEPOOL_OVERRIDES} - echo "galera_repo_host: '${NODEPOOL_MIRROR_HOST}:8080'" >> ${NODEPOOL_OVERRIDES} - echo "nova_virt_type: 'qemu'" >> ${NODEPOOL_OVERRIDES} - - source /etc/os-release - if [[ $ID == 'centos' && VERSION_ID == '7' ]]; then - echo "lxc_centos_package_baseurl: 'http://${NODEPOOL_MIRROR_HOST}:8080/copr-lxc2/epel-7-x86_64/'" >> ${NODEPOOL_OVERRIDES} - echo "lxc_centos_package_key: 'http://${NODEPOOL_MIRROR_HOST}:8080/copr-lxc2/pubkey.gpg'" >> ${NODEPOOL_OVERRIDES} - # NOTE(mnaser): We need to make sure we pull the latest RDO mirror - # which is hashed to avoid cache going stale during CI. - export DLRN_BASE=${DLRN_BASE:-centos7-master/puppet-passed-ci} - rdo_dlrn=`curl --silent ${NODEPOOL_RDO_PROXY}/${DLRN_BASE}/delorean.repo | grep baseurl | cut -d= -f2` - if [[ -z "$rdo_dlrn" ]]; then - echo "Failed to parse dlrn hash" - exit 1 - fi - RDO_MIRROR_HOST=${rdo_dlrn/https:\/\/trunk.rdoproject.org/$NODEPOOL_RDO_PROXY} - echo "openstack_hosts_rdo_repo_url: '${RDO_MIRROR_HOST}'" >> ${NODEPOOL_OVERRIDES} - fi - args: - executable: /bin/bash - tags: - - skip_ansible_lint - - - name: Discover the OpenStack-Infra LXC reverse proxy - shell: | - source /etc/ci/mirror_info.sh - echo ${NODEPOOL_LXC_IMAGE_PROXY} - register: _lxc_mirror - args: - executable: /bin/bash - tags: - - skip_ansible_lint + - name: Configure the OpenStack-Infra mirrors + vars: + nodepool_overrides: + uca_apt_repo_url: "{{ nodepool_vars.NODEPOOL_UCA_MIRROR }}" + openstack_hosts_centos_mirror_url: "{{ nodepool_vars.NODEPOOL_CENTOS_MIRROR }}" + centos_epel_mirror: "{{ nodepool_vars.NODEPOOL_EPEL_MIRROR }}" + galera_repo_host: "{{ nodepool_vars.NODEPOOL_MIRROR_HOST }}:8080" + nova_virt_type: 'qemu' + copy: + dest: /etc/openstack_deploy/user_openstackci.yml + content: "{{ nodepool_overrides | to_nice_yaml }}" - name: Set the package cache timeout to 60 mins in OpenStack-CI set_fact: diff --git a/tests/roles/bootstrap-host/tasks/prepare_networking.yml b/tests/roles/bootstrap-host/tasks/prepare_networking.yml index 9dcfdc8202..02b7dfeb6b 100644 --- a/tests/roles/bootstrap-host/tasks/prepare_networking.yml +++ b/tests/roles/bootstrap-host/tasks/prepare_networking.yml @@ -39,6 +39,7 @@ name: systemd_networkd vars: systemd_networkd_prefix: "osa_testing" + systemd_networkd_epel_mirror: "{{ nodepool_vars is defined | ternary(nodepool_vars.NODEPOOL_EPEL_MIRROR, 'http://download.fedoraproject.org/pub/epel') }}" systemd_interface_cleanup: true systemd_run_networkd: true systemd_netdevs: diff --git a/tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2 b/tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2 index f1c71331b8..42d19194f0 100644 --- a/tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2 +++ b/tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2 @@ -202,10 +202,10 @@ lxc_net_address: 10.255.255.1 lxc_net_netmask: 255.255.255.0 lxc_net_dhcp_range: 10.255.255.2,10.255.255.253 -{% if _lxc_mirror is defined and _lxc_mirror.stdout_lines is defined %} +{% if nodepool_vars is defined and nodepool_vars.NODEPOOL_LXC_IMAGE_PROXY is defined %} ## images.linuxcontainers.org reverse proxy lxc_image_cache_server_mirrors: - - "http://{{ _lxc_mirror.stdout_lines[0] }}" + - "http://{{ nodepool_vars.NODEPOOL_LXC_IMAGE_PROXY }}" {% endif %} {% if cache_timeout is defined %}