Run playbooks out of zuul checkout

Instead of running from system-config, run from the zuul prepared
git repo state. We already have a mutex of one, so we'll never
be fighting. This lets us land stacks of changes and be sure they
will accurately always use the correct git state.

As a todo, we should update manage-projects to do the same with
project-config.

Change-Id: I358554e344e12561d1f3063e0724f6b61d1f15a7

.zuul.yaml

@@ -1464,6 +1464,8 @@
     abstract: true
     semaphore: infra-prod-playbook
     run: playbooks/zuul/run-production-playbook.yaml
+    required-projects:
+      - opendev/system-config
     vars:
       infra_prod_ansible_forks: 5
       infra_prod_playbook_collect_log: false
@@ -1482,19 +1484,10 @@
       playbook_name: zuul_reconfigure.yaml
 
 
-- job:
-    name: infra-prod-update-system-config
-    parent: infra-prod-playbook
-    description: Clone system-config and install modules and roles
-    vars:
-      playbook_name: update-system-config.yaml
-
 - job:
     name: infra-prod-install-ansible
     parent: infra-prod-playbook
     description: Install ansible on bridge
-    dependencies:
-      - infra-prod-update-system-config
     vars:
       playbook_name: install-ansible.yaml
     files:
@@ -1516,7 +1509,6 @@
     parent: infra-prod-playbook
     description: Run the base playbook everywhere
     dependencies:
-      - infra-prod-update-system-config
       - name: infra-prod-install-ansible
         soft: true
     vars:
@@ -1543,7 +1535,6 @@
     vars:
       playbook_name: service-letsencrypt.yaml
     dependencies:
-      - name: infra-prod-update-system-config
       - name: infra-prod-install-ansible
         soft: true
       - name: infra-prod-base
@@ -1577,7 +1568,6 @@
     description: Base job for most service playbooks
     abstract: true
     dependencies:
-      - name: infra-prod-update-system-config
       - name: infra-prod-install-ansible
         soft: true
       - name: infra-prod-base
@@ -1674,7 +1664,6 @@
     parent: infra-prod-service-base
     description: Run service-meetpad.yaml playbook
     dependencies:
-      - name: infra-prod-update-system-config
       - name: infra-prod-install-ansible
         soft: true
       - name: infra-prod-base
@@ -1785,7 +1774,6 @@
     vars:
       playbook_name: service-review.yaml
     dependencies: &infra_prod_service_review_deps
-      - name: infra-prod-update-system-config
       - name: infra-prod-install-ansible
         soft: true
       - name: infra-prod-base
@@ -1824,7 +1812,6 @@
     vars:
       playbook_name: service-gitea.yaml
     dependencies:
-      - name: infra-prod-update-system-config
       - name: infra-prod-install-ansible
         soft: true
       - name: infra-prod-base
@@ -2231,7 +2218,6 @@
         - system-config-promote-image-python-base-3.8
         - system-config-promote-image-python-builder-3.7
         - system-config-promote-image-python-builder-3.8
-        - infra-prod-update-system-config
         - infra-prod-install-ansible
         - infra-prod-base
         - infra-prod-service-letsencrypt
@@ -2249,7 +2235,6 @@
             dependencies:
               - name: system-config-promote-image-gerrit-2.13
                 soft: true
-              - name: infra-prod-update-system-config
               - name: infra-prod-install-ansible
                 soft: true
         - infra-prod-service-bridge
@@ -2292,7 +2277,6 @@
         - tarballs-opendev-goaccess-report
         - zuul-ci-goaccess-report
         # Nightly runs of ansible things for catchup
-        - infra-prod-update-system-config
         - infra-prod-install-ansible
         - infra-prod-base
         - infra-prod-service-letsencrypt
@@ -2311,16 +2295,20 @@
         - infra-prod-remote-puppet-afs
     opendev-prod-hourly:
       jobs:
-        - infra-prod-update-system-config
+        - infra-prod-install-ansible
         - infra-prod-service-nodepool:
             dependencies:
-              - infra-prod-update-system-config
+              - name: infra-prod-install-ansible
+                soft: true
         - infra-prod-service-registry:
             dependencies:
-              - infra-prod-update-system-config
+              - name: infra-prod-install-ansible
+                soft: true
         - infra-prod-service-zuul-preview:
             dependencies:
-              - infra-prod-update-system-config
+              - name: infra-prod-install-ansible
+                soft: true
         - infra-prod-remote-puppet-else:
             dependencies:
-              - infra-prod-update-system-config
+              - name: infra-prod-install-ansible
+                soft: true

playbooks/update-system-config.yaml

@@ -1,10 +0,0 @@
-- hosts: "localhost:!disabled"
-  name: "System-config: Update the system-config repo on bridge"
-  connection: local
-  gather_facts: false
-  tasks:
-    - name: Make sure system-config repo is up to date
-      git:
-        repo: https://opendev.org/opendev/system-config
-        dest: /opt/system-config
-        force: yes

playbooks/zuul/run-production-playbook.yaml

@@ -15,16 +15,19 @@
 
 - hosts: bridge.openstack.org
   tasks:
+    - include_role:
+        name: mirror-workspace-git-repos
+
     - name: Run the production playbook and capture logs
       block:
 
       - name: Log a playbook start header
         become: yes
-        shell: 'echo "Running {{ ansible_date_time.iso8601 }}: ansible-playbook -v -f {{ infra_prod_ansible_forks }} /opt/system-config/playbooks/{{ playbook_name }}" > /var/log/ansible/{{ playbook_name }}.log'
+        shell: 'echo "Running {{ ansible_date_time.iso8601 }}: ansible-playbook -v -f {{ infra_prod_ansible_forks }} /home/zuulcd/src/opendev.org/opendev/system-config/playbooks/{{ playbook_name }}" > /var/log/ansible/{{ playbook_name }}.log'
 
       - name: Run specified playbook on bridge.o.o and redirect output
         become: yes
-        shell: 'ansible-playbook -v -f {{ infra_prod_ansible_forks }} /opt/system-config/playbooks/{{ playbook_name }} >> /var/log/ansible/{{ playbook_name }}.log'
+        shell: 'ansible-playbook -v -f {{ infra_prod_ansible_forks }} /home/zuulcd/src/opendev.org/opendev/system-config/playbooks/{{ playbook_name }} >> /var/log/ansible/{{ playbook_name }}.log'
 
       always: