From b3932f0e8241344ef9ece8c028ebb70bd548defc Mon Sep 17 00:00:00 2001 From: Jonathan Rosser Date: Sat, 7 Mar 2020 15:44:25 +0000 Subject: [PATCH] Use zuul cached repos for openstack services In CI there are cached repos for the openstack services as well as the OSA repos. We are currently using the cached repos for OSA ansible roles, but not for the wider openstack services which are installed from specific git repository SHA as defined by playbooks/defaults/repo_packages/openstack_services.yml. This patch parses the openstack_services.yml file and generates a set of OSA override variables to use the locally cached git repos instead of those at opendev.org. Network traffic is reduced and the impact of OSA jobs on the opendev git servers is eased which has caused problems when the proposal bot makes changes to all OSA repos simultaneously. Depends-On: https://review.opendev.org/712459 Change-Id: I88087e3b4d6a688670e40165f127049956705801 --- scripts/get-ansible-role-requirements.yml | 20 ++++++++ tests/roles/bootstrap-host/tasks/main.yml | 10 ++++ .../templates/user_variables.aio.yml.j2 | 5 ++ zuul.d/jobs.yaml | 47 +++++++++++++++++++ zuul.d/playbooks/pre-osa-aio.yml | 7 ++- 5 files changed, 88 insertions(+), 1 deletion(-) diff --git a/scripts/get-ansible-role-requirements.yml b/scripts/get-ansible-role-requirements.yml index ad83f5901a..81d3988984 100644 --- a/scripts/get-ansible-role-requirements.yml +++ b/scripts/get-ansible-role-requirements.yml @@ -72,6 +72,25 @@ | map(attribute='src') | select('match', 'opendev.org') | list }}" + + # Use the cached repos in the CI images rather than clone from opendev.org + - name: Generate list of openstack service repositories + set_fact: + service_git_repos: "{{ lookup('file', openstack_services_file).splitlines() | + select('contains', 'git_repo') | list | + replace('https://', 'file:///openstack/src/') }}" + + - name: Ensure overrides directory exists + file: + path: '/etc/openstack_deploy' + state: directory + + - name: Create overrides for openstack_services git repos to use local cache + blockinfile: + path: '/etc/openstack_deploy/user_variables_zuulrepos.yml' + block: "{{ service_git_repos | join('\n') }}" + create: yes + when: - "lookup('env', 'ZUUL_SRC_PATH') != ''" @@ -112,6 +131,7 @@ vars: required_roles: "{{ lookup('file', role_file) | from_yaml }}" + openstack_services_file: "{{ playbook_dir }}/../playbooks/defaults/repo_packages/openstack_services.yml" role_file: "{{ playbook_dir }}/../ansible-role-requirements.yml" role_path_default: '/etc/ansible/roles' user_roles: "{{ lookup('file', user_role_path, errors='ignore')|default([], true) | from_yaml }}" diff --git a/tests/roles/bootstrap-host/tasks/main.yml b/tests/roles/bootstrap-host/tasks/main.yml index 6d0e18cb7e..cab28153d6 100644 --- a/tests/roles/bootstrap-host/tasks/main.yml +++ b/tests/roles/bootstrap-host/tasks/main.yml @@ -57,6 +57,16 @@ tags: - prepare-data-disk +# Symlink host repos to /openstack/src to match the lxc continer bind mount +# NOTE(jrosser) this must happen *after* prepare_data_disk as /openstack may remounted +- name: Symlink /openstack/src to /home/zuul/src + file: + src: "{{ lookup('env', 'ZUUL_SRC_PATH') }}" + dest: '/openstack/src' + state: link + when: + - "lookup('env', 'ZUUL_SRC_PATH') != ''" + # Prepare the swap space loopback disk # This is only necessary if there isn't swap already - include: prepare_loopback_swap.yml diff --git a/tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2 b/tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2 index e1de11e6ec..c23e142e68 100644 --- a/tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2 +++ b/tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2 @@ -217,6 +217,11 @@ cache_timeout: {{ cache_timeout }} # AIO build time. Options are: [machinectl, overlayfs, btrfs, zfs, dir, lvm] lxc_container_backing_store: "{{ lxc_container_backing_store }}" +# bind mount the zuul repos into the containers +lxc_container_bind_mounts: + - host_directory: "/home/zuul/src" + container_directory: "/openstack/src" + ## Always setup tempest, the resources for it, then execute tests tempest_install: yes tempest_run: yes diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 6d86af54c3..1b24f299fe 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -21,6 +21,7 @@ Uses the gate-check-commit.sh script, running a default aio deploy. required-projects: + # OSA repos - name: openstack/openstack-ansible - name: openstack/ansible-hardening - name: openstack/openstack-ansible-apt_package_pinning @@ -75,6 +76,52 @@ - name: openstack/ansible-role-systemd_networkd - name: openstack/ansible-role-python_venv_build - name: openstack/ansible-role-uwsgi + # OpenStack service repos - listed here to make cached repos available in CI + - name: openstack/requirements + - name: openstack/aodh + - name: openstack/barbican + - name: openstack/ceilometer + - name: openstack/cinder + - name: openstack/designate + - name: openstack/designate-dashboard + - name: openstack/glance + - name: openstack/heat + - name: openstack/heat-dashboard + - name: openstack/horizon + - name: openstack/ironic-ui + - name: openstack/magnum-ui + - name: openstack/masakari-dashboard + - name: openstack/neutron-fwaas-dashboard + - name: openstack/sahara-dashboard + - name: openstack/keystone + - name: openstack/manila + - name: openstack/neutron + - name: openstack/neutron-vpnaas + - name: openstack/neutron-fwaas + - name: openstack/neutron-dynamic-routing + - name: openstack/networking-odl + - name: openstack/networking-ovn + - name: openstack/networking-bgpvpn + - name: openstack/networking-sfc + - name: openstack/nova + - name: openstack/sahara + - name: openstack/swift + - name: openstack/ironic + - name: openstack/magnum + - name: openstack/masakari + - name: openstack/masakari-monitors + - name: openstack/mistral + - name: openstack/murano + - name: openstack/trove + - name: openstack/trove-dashboard + - name: openstack/octavia + - name: openstack/panko + - name: openstack/placement + - name: openstack/tacker + - name: openstack/congress + - name: openstack/octavia-dashboard + - name: openstack/blazar + - name: openstack/blazar-nova pre-run: - zuul.d/playbooks/pre-gate-cleanup.yml - zuul.d/playbooks/pre-gate-scenario.yml diff --git a/zuul.d/playbooks/pre-osa-aio.yml b/zuul.d/playbooks/pre-osa-aio.yml index 30c92b48e6..e6ac4b068f 100644 --- a/zuul.d/playbooks/pre-osa-aio.yml +++ b/zuul.d/playbooks/pre-osa-aio.yml @@ -17,6 +17,8 @@ hosts: all[0] vars_files: - "{{ zuul.executor.work_root }}/osa-gate-scenario.yml" + vars: + _zuul_src_path: "{{ ansible_user_dir }}/src" tasks: - name: Run bootstrap-ansible script become: yes @@ -28,7 +30,7 @@ environment: # ZUUL_SRC_PATH is used by tests/get-ansible-role-requirements to # where the CI provided git sources were cloned. - ZUUL_SRC_PATH: "{{ ansible_user_dir }}/src" + ZUUL_SRC_PATH: "{{ _zuul_src_path }}" ANSIBLE_PACKAGE: "{{ ansible_package | default('') }}" SETUP_ARA: 'true' when: action != 'upgrade' @@ -40,6 +42,9 @@ chdir: "src/opendev.org/openstack/openstack-ansible" executable: /bin/bash environment: + # ZUUL_SRC_PATH is used by tests/roles/bootstrap-host + # symlink the zuul roles into /openstack/src + ZUUL_SRC_PATH: "{{ _zuul_src_path }}" SCENARIO: "{{ scenario }}" ACTION: "{{ action }}" INSTALL_METHOD: "{{ install_method }}"