Use zuul cached repos for openstack services

In CI there are cached repos for the openstack services as well as
the OSA repos. We are currently using the cached repos for OSA ansible
roles, but not for the wider openstack services which are installed
from specific git repository SHA as defined by
playbooks/defaults/repo_packages/openstack_services.yml.

This patch parses the openstack_services.yml file and generates a
set of OSA override variables to use the locally cached git repos
instead of those at opendev.org. Network traffic is reduced and the
impact of OSA jobs on the opendev git servers is eased which has
caused problems when the proposal bot makes changes to all OSA repos
simultaneously.

Depends-On: https://review.opendev.org/712459
Change-Id: I88087e3b4d6a688670e40165f127049956705801

scripts/get-ansible-role-requirements.yml

@@ -72,6 +72,25 @@
                                                    | map(attribute='src')
                                                    | select('match', 'opendev.org')
                                                    | list }}"
+
+        # Use the cached repos in the CI images rather than clone from opendev.org
+        - name: Generate list of openstack service repositories
+          set_fact:
+            service_git_repos: "{{ lookup('file', openstack_services_file).splitlines() |
+                                   select('contains', 'git_repo') | list |
+                                   replace('https://', 'file:///openstack/src/') }}"
+
+        - name: Ensure overrides directory exists
+          file:
+            path: '/etc/openstack_deploy'
+            state: directory
+
+        - name: Create overrides for openstack_services git repos to use local cache
+          blockinfile:
+            path: '/etc/openstack_deploy/user_variables_zuulrepos.yml'
+            block: "{{ service_git_repos | join('\n') }}"
+            create: yes
+
       when:
         - "lookup('env', 'ZUUL_SRC_PATH') != ''"
 
@@ -112,6 +131,7 @@
 
   vars:
     required_roles: "{{ lookup('file', role_file) | from_yaml }}"
+    openstack_services_file: "{{ playbook_dir }}/../playbooks/defaults/repo_packages/openstack_services.yml"
     role_file: "{{ playbook_dir }}/../ansible-role-requirements.yml"
     role_path_default: '/etc/ansible/roles'
     user_roles: "{{ lookup('file', user_role_path, errors='ignore')|default([], true) | from_yaml }}"

tests/roles/bootstrap-host/tasks/main.yml

@@ -57,6 +57,16 @@
   tags:
     - prepare-data-disk
 
+# Symlink host repos to /openstack/src to match the lxc continer bind mount
+# NOTE(jrosser) this must happen *after* prepare_data_disk as /openstack may remounted
+- name: Symlink /openstack/src to /home/zuul/src
+  file:
+    src: "{{ lookup('env', 'ZUUL_SRC_PATH') }}"
+    dest: '/openstack/src'
+    state: link
+  when:
+    - "lookup('env', 'ZUUL_SRC_PATH') != ''"
+
 # Prepare the swap space loopback disk
 # This is only necessary if there isn't swap already
 - include: prepare_loopback_swap.yml

tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2

@@ -217,6 +217,11 @@ cache_timeout: {{ cache_timeout }}
 # AIO build time. Options are: [machinectl, overlayfs, btrfs, zfs, dir, lvm]
 lxc_container_backing_store: "{{ lxc_container_backing_store }}"
 
+# bind mount the zuul repos into the containers
+lxc_container_bind_mounts:
+  - host_directory: "/home/zuul/src"
+    container_directory: "/openstack/src"
+
 ## Always setup tempest, the resources for it, then execute tests
 tempest_install: yes
 tempest_run: yes

zuul.d/jobs.yaml

@@ -21,6 +21,7 @@
       Uses the gate-check-commit.sh script, running a default
       aio deploy.
     required-projects:
+      # OSA repos
       - name: openstack/openstack-ansible
       - name: openstack/ansible-hardening
       - name: openstack/openstack-ansible-apt_package_pinning
@@ -75,6 +76,52 @@
       - name: openstack/ansible-role-systemd_networkd
       - name: openstack/ansible-role-python_venv_build
       - name: openstack/ansible-role-uwsgi
+      # OpenStack service repos - listed here to make cached repos available in CI
+      - name: openstack/requirements
+      - name: openstack/aodh
+      - name: openstack/barbican
+      - name: openstack/ceilometer
+      - name: openstack/cinder
+      - name: openstack/designate
+      - name: openstack/designate-dashboard
+      - name: openstack/glance
+      - name: openstack/heat
+      - name: openstack/heat-dashboard
+      - name: openstack/horizon
+      - name: openstack/ironic-ui
+      - name: openstack/magnum-ui
+      - name: openstack/masakari-dashboard
+      - name: openstack/neutron-fwaas-dashboard
+      - name: openstack/sahara-dashboard
+      - name: openstack/keystone
+      - name: openstack/manila
+      - name: openstack/neutron
+      - name: openstack/neutron-vpnaas
+      - name: openstack/neutron-fwaas
+      - name: openstack/neutron-dynamic-routing
+      - name: openstack/networking-odl
+      - name: openstack/networking-ovn
+      - name: openstack/networking-bgpvpn
+      - name: openstack/networking-sfc
+      - name: openstack/nova
+      - name: openstack/sahara
+      - name: openstack/swift
+      - name: openstack/ironic
+      - name: openstack/magnum
+      - name: openstack/masakari
+      - name: openstack/masakari-monitors
+      - name: openstack/mistral
+      - name: openstack/murano
+      - name: openstack/trove
+      - name: openstack/trove-dashboard
+      - name: openstack/octavia
+      - name: openstack/panko
+      - name: openstack/placement
+      - name: openstack/tacker
+      - name: openstack/congress
+      - name: openstack/octavia-dashboard
+      - name: openstack/blazar
+      - name: openstack/blazar-nova
     pre-run:
       - zuul.d/playbooks/pre-gate-cleanup.yml
       - zuul.d/playbooks/pre-gate-scenario.yml

zuul.d/playbooks/pre-osa-aio.yml

@@ -17,6 +17,8 @@
   hosts: all[0]
   vars_files:
     - "{{ zuul.executor.work_root }}/osa-gate-scenario.yml"
+  vars:
+    _zuul_src_path: "{{ ansible_user_dir }}/src"
   tasks:
     - name: Run bootstrap-ansible script
       become: yes
@@ -28,7 +30,7 @@
       environment:
         # ZUUL_SRC_PATH is used by tests/get-ansible-role-requirements to
         # where the CI provided git sources were cloned.
-        ZUUL_SRC_PATH: "{{ ansible_user_dir }}/src"
+        ZUUL_SRC_PATH: "{{ _zuul_src_path }}"
         ANSIBLE_PACKAGE: "{{ ansible_package | default('') }}"
         SETUP_ARA: 'true'
       when: action != 'upgrade'
@@ -40,6 +42,9 @@
         chdir: "src/opendev.org/openstack/openstack-ansible"
         executable: /bin/bash
       environment:
+        # ZUUL_SRC_PATH is used by tests/roles/bootstrap-host
+        # symlink the zuul roles into /openstack/src
+        ZUUL_SRC_PATH: "{{ _zuul_src_path }}"
         SCENARIO: "{{ scenario }}"
         ACTION: "{{ action }}"
         INSTALL_METHOD: "{{ install_method }}"