Use project-config from zuul instead of direct clones

We use project-config for gerrit, gitea and nodepool config. That's
cool, because can clone that from zuul too and make sure that each
prod run we're doing runs with the contents of the patch in question.

Introduce a flag file that can be touched in /home/zuulcd that will
block zuul from running prod playbooks. By default, if the file is
there, zuul will wait for an hour before giving up.

Rename zuulcd to zuul

To better align prod and test, name the zuul user zuul.

Change-Id: I83c38c9c430218059579f3763e02d6b9f40c7b89

.zuul.yaml

@@ -1069,6 +1069,9 @@
           label: ubuntu-xenial
         - name: nb01-test.opendev.org
           label: ubuntu-bionic
+    required-projects:
+      - openstack/project-config
+      - opendev/system-config
     vars:
       run_playbooks:
         - playbooks/service-letsencrypt.yaml
@@ -1279,6 +1282,9 @@
           label: ubuntu-bionic
         - name: gitea99.opendev.org
           label: ubuntu-bionic
+    required-projects:
+      - openstack/project-config
+      - opendev/system-config
     vars:
       run_playbooks:
         - playbooks/service-letsencrypt.yaml
@@ -1384,6 +1390,9 @@
           label: ubuntu-xenial
         - name: review-dev01.opendev.org
           label: ubuntu-xenial
+    required-projects:
+      - openstack/project-config
+      - opendev/system-config
     vars:
       run_playbooks:
         - playbooks/service-letsencrypt.yaml
@@ -1460,7 +1469,8 @@
       This is a parent job designed to be inherited to enabled
       CD deployment of our infrastructure. Set playbook_name to
       specify the playbook relative to
-      bridge.openstack.org:/opt/system-config/playbooks
+      /home/zuul/src/opendev.org/opendev/system-config/playbooks
+      on bridge.openstack.org.
     abstract: true
     semaphore: infra-prod-playbook
     run: playbooks/zuul/run-production-playbook.yaml
@@ -1557,6 +1567,9 @@
     allowed-projects:
       - opendev/system-config
       - openstack/project-config
+    required-projects:
+      - opendev/system-config
+      - openstack/project-config
     vars:
       playbook_name: manage-projects.yaml
       infra_prod_ansible_forks: 10
@@ -1631,6 +1644,9 @@
     description: Run service-nodepool.yaml playbook
     vars:
       playbook_name: service-nodepool.yaml
+    required-projects:
+      - opendev/system-config
+      - openstack/project-config
     files:
       - inventory/.*
       - playbooks/service-nodepool.yaml

ansible.cfg

@@ -1,33 +0,0 @@
-# This ansible.cfg file is only for running ad-hoc commands from
-# the /opt/system-config checkout. This file should be kept in
-# sync with playbooks/roles/install-ansible/templates/ansible.cfg.j2
-[defaults]
-inventory=/opt/system-config/inventory/openstack.yaml,/opt/system-config/inventory/groups.yaml,/etc/ansible/hosts/emergency.yaml
-library=/usr/share/ansible
-log_path=/var/log/ansible/ansible.log
-inventory_plugins=/opt/system-config/playbooks/roles/install-ansible/files/inventory_plugins/inventory_plugins
-roles_path=/opt/system-config/roles:/etc/ansible/roles
-retry_files_enabled=False
-retry_files_save_path=
-gathering=smart
-fact_caching=jsonfile
-fact_caching_connection=/var/cache/ansible/facts
-# Squash warning about ansible auto-transforming group names with -'s in them
-force_valid_group_names=ignore
-callback_whitelist=profile_tasks, timer
-callback_plugins=/etc/ansible/callback_plugins
-stdout_callback=debug
-
-[inventory]
-enable_plugins=yaml,yamlgroup,advanced_host_list,ini
-cache=True
-cache_plugin=jsonfile
-cache_connection=/var/cache/ansible/inventory
-any_unparsed_is_failed=True
-
-[ssh_connection]
-retries=3
-pipelining = True
-
-[callback_profile_tasks]
-task_output_limit = 50

doc/source/afs.rst

@@ -441,7 +441,7 @@ read-write volumes.
 
   .. code-block:: console
 
-    root@bridge:~# /opt/system-config/tools/hieraedit.py \
+    root@bridge:~# /home/zuul/src/opendev.org/opendev/system-config/tools/hieraedit.py \
       --yaml /etc/ansible/hosts/host_vars/mirror-update01.opendev.org.yaml \
       -f /path/to/foo.keytab KEYNAME
 

launch/launch-node.py

@@ -177,8 +177,8 @@ def bootstrap_server(server, key, name, volume_device, keep,
         t.start()
 
         inventory_list = (
-            '/opt/system-config/inventory/openstack.yaml',
-            '/opt/system-config/inventory/groups.yaml',
+            '/etc/ansible/hosts/openstack.yaml',
+            '/etc/ansible/hosts/groups.yaml',
             '/etc/ansible/hosts/emergency.yaml',
             jobdir.inventory_root,
         )

playbooks/group_vars/all.yaml

@@ -28,6 +28,8 @@ iptables_base_public_udp_ports: []
 iptables_extra_public_udp_ports: []
 iptables_public_udp_ports: "{{ iptables_base_public_udp_ports + iptables_extra_public_udp_ports }}"
 
+project_config_src: /home/zuul/src/opendev.org/openstack/project-config
+
 # When adding new users, always pick a UID larger than the last UID, do not
 # fill in holes in the middle of the range.
 all_users:
@@ -150,7 +152,7 @@ all_users:
     uid: 2030
     gid: 2030
 
-  zuulcd:
+  zuul:
     comment: Zuul CICD
     key: |
       ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcXd/QJDEprSLh6N6bULnhchf9M+uzYBEJ2b51Au67FON+5M6VEj5Ut+DlkEPhabOP+tSv9Cn1HpmpBjdEOXdmBj6JS7G/gBb4w28oZDyNjrPT2ebpRw/XnVEkGfikR2J+j3o7CV+ybhLDalXm2TUDReVXnONUq3YzZbjRzoYs0xxrxyss47vZP0xFpsAt9jCMAJW2k6H589VUY38k9LFyhZUZ72FB6eJ68B9GN0TimBYm2DqvupBGQrRhkP8OZ0WoBV8PulKXaHVFdmfBNHB7E7FLlZKuiM6nkV4bOWMGOB/TF++wXBK86t9po3pWCM7+kr72xGRTE+6LuZ2z1K+h zuul-system-config-20180924

playbooks/group_vars/puppet.yaml

@@ -5,7 +5,7 @@ puppet_reports: none
 manage_config: true
 manifest: /opt/system-config/production/manifests/site.pp
 manifest_base: /opt/system-config/production
-mgmt_manifestpath: /opt/system-config/
+mgmt_manifestpath: /home/zuul/src/opendev.org/opendev/system-config/
 puppet_logdest: syslog
 mgmt_hieradata: /etc/ansible/hosts
 mgmt_puppet_module_dir: /etc/puppet/modules

playbooks/host_vars/bridge.openstack.org.yaml

@@ -2,4 +2,4 @@ ansible_python_interpreter: python3
 bastion_key_exclusive: false
 kube_config_template: clouds/bridge_kube_config.yaml.j2
 extra_users:
-  - zuulcd
+  - zuul

playbooks/host_vars/review-dev01.opendev.org.yaml

@@ -10,6 +10,6 @@ letsencrypt_gid: 3001
 gerrit_storyboard_url: https://storyboard-dev.openstack.org
 gerrit_vhost_name: review-dev.opendev.org
 gerrit_redirect_vhost: review-dev.openstack.org
-gerrit_project_config_base: /opt/project-config/dev
 gerrit_project_creator_user: openstack-dev-project-creator
 gerrit_self_hostkey: '[review-dev.opendev.org]:29418,[review-dev.openstack.org]:29418,[23.253.109.153]:29418,[2001:4800:7819:104:be76:4eff:fe04:8e55]:29418 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC4J4BJ/C6kl1PcfD5ZdpYIwWXA+vRiB4USncZQHW9+Idtdr4dZRA05RlBAfiTkKKhjarJpt8PQP2hYt8aJL1miZZjp1s05d9mxGVHfoH7Vyg85vhRa7Jg4VZS0cu34R909q23cBcjSNQSyVKP9neOqovoV/DyB8HHEg0kbsOWC3qzdA+6aVdVV7Mtx/0t0MyiTz0xA5ZCRFwF6IuiMPHLNk128qDhjO2UXnrhyP5A7Kl/JHpIWToLKGIorePndFcFyNXlWIhBoQRDcX6FYjPdavjAGlK1S/Jd5DVJ184Z7rEXL682o487c0NQ/lAV4QF3iz0Aw9QRVrUw21xWvfU4R'
+project_config_subdir: dev/

playbooks/manage-projects.yaml

@@ -1,16 +1,3 @@
-# Run on localhost for lookup plugins, on review/review-dev
-# because manage-projects runs remotely.
-- hosts: "localhost:!disabled"
-  name: "Clone project-config for projects list"
-  strategy: free
-  connection: local
-  tasks:
-    - name: Clone project-config repo
-      git:
-        repo: https://opendev.org/openstack/project-config
-        dest: /opt/project-config
-        force: yes
-
 - hosts: "gitea:!disabled"
   name: "Create repos on gitea servers"
   strategy: free
@@ -21,11 +8,9 @@
 - hosts: "review:review-dev:!disabled"
   name: "Create repos on gerrit servers"
   tasks:
-    - name: Clone project-config repo
-      git:
-        repo: https://opendev.org/openstack/project-config
-        dest: /opt/project-config
-        force: yes
+    - name: Sync project-config
+      include_role:
+        name: sync-project-config
     - name: Run manage-projects
       include_role:
         name: gerrit

playbooks/roles/cloud-launcher-cron/tasks/main.yaml

@@ -8,7 +8,7 @@
     name: run_cloud_launcher.sh
     state: present
     disabled: "{{ cloud_launcher_disable_job }}"
-    job: '/usr/bin/flock -n /var/run/ansible/run_cloud_launcher.lock /bin/bash /opt/system-config/run_cloud_launcher.sh -c >> /var/log/ansible/run_cloud_launcher_cron.log 2>&1'
+    job: '/usr/bin/flock -n /var/run/ansible/run_cloud_launcher.lock /bin/bash /home/zuul/src/opendev.org/opendev/system-config/run_cloud_launcher.sh -c >> /var/log/ansible/run_cloud_launcher_cron.log 2>&1'
     minute: "{{ cloud_launcher_cron_interval.minute }}"
     hour: "{{ cloud_launcher_cron_interval.hour }}"
     day: "{{ cloud_launcher_cron_interval.day }}"

playbooks/roles/gerrit/defaults/main.yaml

@@ -17,9 +17,8 @@ gerrit_container_volumes:
   - /home/gerrit2/review_site/static:/var/gerrit/static
   - /home/gerrit2/.launchpadlib:/var/gerrit/.launchpadlib
   - /home/gerrit2/.ssh:/var/gerrit/.ssh
-  - '{{ gerrit_project_config_base }}/gerrit/projects.yaml:/var/gerrit/etc/projects.yaml'
-  - '{{ gerrit_project_config_base }}/gerrit/projects.ini:/var/gerrit/etc/projects.ini'
+  - /opt/project-config/gerrit/projects.yaml:/var/gerrit/etc/projects.yaml
+  - /opt/project-config/gerrit/projects.ini:/var/gerrit/etc/projects.ini
 gerrit_database_type: MYSQL
-gerrit_project_config_base: /opt/project-config
 gerrit_project_creator_user: openstack-project-creator
 gerrit_manage_projects_args: "-v"

playbooks/roles/gerrit/tasks/main.yaml

@@ -1,13 +1,6 @@
-# TODO(mordred) We should do *something* where this could use a zuul cloned
-# copy of project-config instead. This is needed not just for things like
-# manage-projects (which could be run completely differently and non-locally)
-# but also for things like notify-impact, which is currently run by a gerrit
-# hook inside of the container via jeepyb.
-- name: Clone project-config repo
-  git:
-    repo: https://opendev.org/openstack/project-config
-    dest: /opt/project-config
-    force: yes
+- name: Sync project-config
+  include_role:
+    name: sync-project-config
 
 - name: Ensure /etc/gerrit-compose directory
   file:
@@ -203,7 +196,7 @@
 
 - name: Copy notify-impact yaml file
   copy:
-    src: "{{ gerrit_project_config_base }}/gerrit/notify_impact.yaml"
+    src: "/opt/project-config/gerrit/notify_impact.yaml"
     dest: "{{ gerrit_site_dir }}/hooks/notify_impact.yaml"
     remote_src: yes
     owner: "{{ gerrit_user_name }}"

playbooks/roles/gerrit/templates/manage-projects.j2

@@ -15,9 +15,9 @@
 # limitations under the License.
 
 exec docker run --rm --net=host -u root \
-  -v{{ gerrit_project_config_base }}:/opt/project-config \
-  -v{{ gerrit_project_config_base }}/gerrit/acls:/home/gerrit2/acls \
-  -v{{ gerrit_project_config_base }}/gerrit/projects.yaml:/home/gerrit2/projects.yaml \
+  -v/opt/project-config:/opt/project-config \
+  -v/opt/project-config/gerrit/acls:/home/gerrit2/acls \
+  -v/opt/project-config/gerrit/projects.yaml:/home/gerrit2/projects.yaml \
   -v/opt/lib/git:/opt/lib/git \
   -v/opt/lib/jeepyb:/opt/lib/jeepyb \
   -v/home/gerrit2/review_site/etc/ssh_project_rsa_key:/home/gerrit2/review_site/etc/ssh_project_rsa_key \

playbooks/roles/gerrit/templates/track-upstream.j2

@@ -15,8 +15,8 @@
 # limitations under the License.
 
 exec docker run --rm --net=host -u root \
-  -v{{ gerrit_project_config_base }}:/opt/project-config \
-  -v{{ gerrit_project_config_base }}/gerrit/projects.yaml:/home/gerrit2/projects.yaml \
+  -v/opt/project-config:/opt/project-config \
+  -v/opt/project-config/gerrit/projects.yaml:/home/gerrit2/projects.yaml \
   -v/opt/lib/git:/opt/lib/git \
   -v/opt/lib/jeepyb:/opt/lib/jeepyb \
   -v/home/gerrit2/review_site/etc/ssh_project_rsa_key:/home/gerrit2/review_site/etc/ssh_project_rsa_key \

playbooks/roles/gitea-git-repos/tasks/main.yaml

@@ -4,5 +4,5 @@
     password: "{{ gitea_root_password }}"
     always_update: "{{ gitea_always_update }}"
     # Lookup runs locally on the calling machine, so doesn't need
-    # /opt/project-config remotely
-    projects: "{{ lookup('file', '/opt/project-config/gerrit/projects.yaml') | from_yaml }}"
+    # project-config remotely
+    projects: "{{ lookup('file', project_config_src + '/gerrit/projects.yaml') | from_yaml }}"

playbooks/roles/nodepool-base/tasks/main.yaml

@@ -18,16 +18,9 @@
     name: install-zookeeper
   when: nodepool_base_install_zookeeper
 
-# NOTE(ianw) : A note on testing; we have some configurations for
-# system-config-run-nodepool test hosts committed to project-config.
-# Since this is a protected repo we can't speculatively test, which is
-# why we're just cloning from opendev.org master and not a local
-# checkout here.  We don't expect the configs to change so this is OK.
-- name: Clone the project-config repo for configs
-  git:
-    repo: 'https://opendev.org/openstack/project-config'
-    dest: /opt/project-config
-    force: yes
+- name: Sync project-config
+  include_role:
+    name: sync-project-config
 
 - name: Create nodepool config dir
   file:
@@ -52,4 +45,4 @@
   file:
     state: link
     src: /opt/project-config/nodepool/elements
-    dest: /etc/nodepool/elements
+    dest: /etc/nodepool/elements

playbooks/roles/sync-project-config/README.rst

@@ -0,0 +1 @@
+Sync project-config to remote host

playbooks/roles/sync-project-config/defaults/main.yaml

@@ -0,0 +1,2 @@
+project_config_dest: /opt/project-config
+project_config_subdir: ""

playbooks/roles/sync-project-config/tasks/main.yaml

@@ -0,0 +1,11 @@
+- name: Create project-config dir
+  file:
+    path: '{{ project_config_dest }}'
+    state: directory
+
+- name: Sync project-config repo
+  synchronize:
+    src: '{{ project_config_src }}/{{ project_config_subdir }}'
+    dest: '{{ project_config_dest }}'
+
+

playbooks/zuul/run-base.yaml

@@ -15,11 +15,6 @@
         write_inventory_exclude_hostvars:
           - ansible_user
           - ansible_python_interpreter
-    - name: Set up /opt/system-config repo
-      git:
-        repo: /home/zuul/src/opendev.org/opendev/system-config
-        dest: /opt/system-config
-        force: yes
     - name: Add groups config for test nodes
       template:
         src: "templates/gate-groups.yaml.j2"
@@ -73,6 +68,28 @@
         - host_vars/nb01-test.opendev.org.yaml
     - name: Display group membership
       command: ansible localhost -m debug -a 'var=groups'
+
+    # In prod, bridge installs a zuul user, but in zuul we already have a zuul user, so we really need 
+    # to not modify it.
+    - name: Load bridge hostvars
+      slurp:
+        path: /home/zuul/src/opendev.org/opendev/system-config/playbooks/host_vars/bridge.openstack.org.yaml
+      register: bridge_hostvar_content
+    - name: Parse bridge_hostvars
+      set_fact:
+        bridge_hostvars: "{{ bridge_hostvar_content.content | b64decode | from_yaml }}"
+    - name: Overwrite extra_users
+      vars:
+        new_config:
+          extra_users: []
+      set_fact:
+        bridge_hostvars: "{{ bridge_hostvars | combine(new_config) }}"
+    - name: Save bridge hostvars
+      copy:
+        content: "{{ bridge_hostvars | to_nice_yaml }}"
+        dest: /home/zuul/src/opendev.org/opendev/system-config/playbooks/host_vars/bridge.openstack.org.yaml
+      become: true
+
     - name: Run base.yaml
       command: ansible-playbook -v /home/zuul/src/opendev.org/opendev/system-config/playbooks/base.yaml
     - name: Run bridge service playbook

playbooks/zuul/run-production-playbook.yaml

@@ -4,7 +4,7 @@
       add_host:
         name: bridge.openstack.org
         ansible_python_interpreter: python3
-        ansible_user: zuulcd
+        ansible_user: zuul
 
 - hosts: localhost
   tasks:
@@ -15,6 +15,13 @@
 
 - hosts: bridge.openstack.org
   tasks:
+    - name: Make sure a manaul maint isn't going on
+      wait_for:
+        path: /home/zuul/DISABLE-ANSIBLE
+        state: absent
+        sleep: 10
+        timeout: 3600 # Wait for an hour before bailing
+
     - name: Synchronize src repos to workspace directory.
       synchronize:
         delete: false
@@ -28,11 +35,11 @@
 
       - name: Log a playbook start header
         become: yes
-        shell: 'echo "Running {{ ansible_date_time.iso8601 }}: ansible-playbook -v -f {{ infra_prod_ansible_forks }} /home/zuulcd/src/opendev.org/opendev/system-config/playbooks/{{ playbook_name }}" > /var/log/ansible/{{ playbook_name }}.log'
+        shell: 'echo "Running {{ ansible_date_time.iso8601 }}: ansible-playbook -v -f {{ infra_prod_ansible_forks }} /home/zuul/src/opendev.org/opendev/system-config/playbooks/{{ playbook_name }}" > /var/log/ansible/{{ playbook_name }}.log'
 
       - name: Run specified playbook on bridge.o.o and redirect output
         become: yes
-        shell: 'ansible-playbook -v -f {{ infra_prod_ansible_forks }} /home/zuulcd/src/opendev.org/opendev/system-config/playbooks/{{ playbook_name }} >> /var/log/ansible/{{ playbook_name }}.log'
+        shell: 'ansible-playbook -v -f {{ infra_prod_ansible_forks }} /home/zuul/src/opendev.org/opendev/system-config/playbooks/{{ playbook_name }} >> /var/log/ansible/{{ playbook_name }}.log'
 
       always:
 

run_cloud_launcher.sh

@@ -19,7 +19,7 @@
 # expect.
 set -e
 export ANSIBLE_LOG_PATH=/var/log/puppet_run_cloud_launcher.log
-SYSTEM_CONFIG=/opt/system-config
+SYSTEM_CONFIG=/home/zuul/src/opendev.org/opendev/system-config
 ANSIBLE_PLAYBOOKS=$SYSTEM_CONFIG/playbooks
 
 # It's possible for connectivity to a server or manifest application to break

testinfra/test_bridge.py

@@ -79,8 +79,8 @@ def test_kubectl(host):
     assert kube.rc == 0
 
 
-def test_zuulcd_authorized_keys(host):
-    authorized_keys = host.file('/home/zuulcd/.ssh/authorized_keys')
+def test_zuul_authorized_keys(host):
+    authorized_keys = host.file('/home/zuul/.ssh/authorized_keys')
     assert authorized_keys.exists
 
     content = authorized_keys.content.decode('utf8')

tools/kick.sh

@@ -16,5 +16,5 @@
 
 for playbook in base.yaml remote_puppet_adhoc.yaml ; do
     ansible-playbook -f1 --limit $1 \
-        /opt/system-config/playbooks/$playbook
+        /home/zuul/src/opendev.org/opendev/system-config/playbooks/$playbook
 done