Merge "Use zuul cached repos for openstack services"

scripts/get-ansible-role-requirements.yml

@@ -72,6 +72,25 @@
                                                    | map(attribute='src')
                                                    | select('match', 'opendev.org')
                                                    | list }}"
+
+        # Use the cached repos in the CI images rather than clone from opendev.org
+        - name: Generate list of openstack service repositories
+          set_fact:
+            service_git_repos: "{{ lookup('file', openstack_services_file).splitlines() |
+                                   select('contains', 'git_repo') | list |
+                                   replace('https://', 'file:///openstack/src/') }}"
+
+        - name: Ensure overrides directory exists
+          file:
+            path: '/etc/openstack_deploy'
+            state: directory
+
+        - name: Create overrides for openstack_services git repos to use local cache
+          blockinfile:
+            path: '/etc/openstack_deploy/user_variables_zuulrepos.yml'
+            block: "{{ service_git_repos | join('\n') }}"
+            create: yes
+
       when:
         - "lookup('env', 'ZUUL_SRC_PATH') != ''"
 
@@ -112,6 +131,7 @@
 
   vars:
     required_roles: "{{ lookup('file', role_file) | from_yaml }}"
+    openstack_services_file: "{{ playbook_dir }}/../playbooks/defaults/repo_packages/openstack_services.yml"
     role_file: "{{ playbook_dir }}/../ansible-role-requirements.yml"
     role_path_default: '/etc/ansible/roles'
     user_roles: "{{ lookup('file', user_role_path, errors='ignore')|default([], true) | from_yaml }}"

tests/roles/bootstrap-host/tasks/main.yml

@@ -57,6 +57,16 @@
   tags:
     - prepare-data-disk
 
+# Symlink host repos to /openstack/src to match the lxc continer bind mount
+# NOTE(jrosser) this must happen *after* prepare_data_disk as /openstack may remounted
+- name: Symlink /openstack/src to /home/zuul/src
+  file:
+    src: "{{ lookup('env', 'ZUUL_SRC_PATH') }}"
+    dest: '/openstack/src'
+    state: link
+  when:
+    - "lookup('env', 'ZUUL_SRC_PATH') != ''"
+
 # Prepare the swap space loopback disk
 # This is only necessary if there isn't swap already
 - include: prepare_loopback_swap.yml

tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2

@@ -217,6 +217,11 @@ cache_timeout: {{ cache_timeout }}
 # AIO build time. Options are: [machinectl, overlayfs, btrfs, zfs, dir, lvm]
 lxc_container_backing_store: "{{ lxc_container_backing_store }}"
 
+# bind mount the zuul repos into the containers
+lxc_container_bind_mounts:
+  - host_directory: "/home/zuul/src"
+    container_directory: "/openstack/src"
+
 ## Always setup tempest, the resources for it, then execute tests
 tempest_install: yes
 tempest_run: yes

zuul.d/jobs.yaml

@@ -21,6 +21,7 @@
       Uses the gate-check-commit.sh script, running a default
       aio deploy.
     required-projects:
+      # OSA repos
       - name: openstack/openstack-ansible
       - name: openstack/ansible-hardening
       - name: openstack/openstack-ansible-apt_package_pinning
@@ -75,6 +76,52 @@
       - name: openstack/ansible-role-systemd_networkd
       - name: openstack/ansible-role-python_venv_build
       - name: openstack/ansible-role-uwsgi
+      # OpenStack service repos - listed here to make cached repos available in CI
+      - name: openstack/requirements
+      - name: openstack/aodh
+      - name: openstack/barbican
+      - name: openstack/ceilometer
+      - name: openstack/cinder
+      - name: openstack/designate
+      - name: openstack/designate-dashboard
+      - name: openstack/glance
+      - name: openstack/heat
+      - name: openstack/heat-dashboard
+      - name: openstack/horizon
+      - name: openstack/ironic-ui
+      - name: openstack/magnum-ui
+      - name: openstack/masakari-dashboard
+      - name: openstack/neutron-fwaas-dashboard
+      - name: openstack/sahara-dashboard
+      - name: openstack/keystone
+      - name: openstack/manila
+      - name: openstack/neutron
+      - name: openstack/neutron-vpnaas
+      - name: openstack/neutron-fwaas
+      - name: openstack/neutron-dynamic-routing
+      - name: openstack/networking-odl
+      - name: openstack/networking-ovn
+      - name: openstack/networking-bgpvpn
+      - name: openstack/networking-sfc
+      - name: openstack/nova
+      - name: openstack/sahara
+      - name: openstack/swift
+      - name: openstack/ironic
+      - name: openstack/magnum
+      - name: openstack/masakari
+      - name: openstack/masakari-monitors
+      - name: openstack/mistral
+      - name: openstack/murano
+      - name: openstack/trove
+      - name: openstack/trove-dashboard
+      - name: openstack/octavia
+      - name: openstack/panko
+      - name: openstack/placement
+      - name: openstack/tacker
+      - name: openstack/congress
+      - name: openstack/octavia-dashboard
+      - name: openstack/blazar
+      - name: openstack/blazar-nova
     pre-run:
       - zuul.d/playbooks/pre-gate-cleanup.yml
       - zuul.d/playbooks/pre-gate-scenario.yml

zuul.d/playbooks/pre-osa-aio.yml

@@ -17,6 +17,8 @@
   hosts: all[0]
   vars_files:
     - "{{ zuul.executor.work_root }}/osa-gate-scenario.yml"
+  vars:
+    _zuul_src_path: "{{ ansible_user_dir }}/src"
   tasks:
     - name: Run bootstrap-ansible script
       become: yes
@@ -28,7 +30,7 @@
       environment:
         # ZUUL_SRC_PATH is used by tests/get-ansible-role-requirements to
         # where the CI provided git sources were cloned.
-        ZUUL_SRC_PATH: "{{ ansible_user_dir }}/src"
+        ZUUL_SRC_PATH: "{{ _zuul_src_path }}"
         ANSIBLE_PACKAGE: "{{ ansible_package | default('') }}"
         SETUP_ARA: 'true'
       when: action != 'upgrade'
@@ -40,6 +42,9 @@
         chdir: "src/opendev.org/openstack/openstack-ansible"
         executable: /bin/bash
       environment:
+        # ZUUL_SRC_PATH is used by tests/roles/bootstrap-host
+        # symlink the zuul roles into /openstack/src
+        ZUUL_SRC_PATH: "{{ _zuul_src_path }}"
         SCENARIO: "{{ scenario }}"
         ACTION: "{{ action }}"
         INSTALL_METHOD: "{{ install_method }}"