From ca684c775ba56224294ea0b651134bc7001fabe5 Mon Sep 17 00:00:00 2001
From: Monty Taylor <mordred@inaugust.com>
Date: Sat, 18 Apr 2020 08:47:56 -0500
Subject: [PATCH] Move puppet apply jobs to system-config repo

These use legacy-base, which sucks, but what sucks even more is
that they are in openstack-zuul-jobs, which makes them extra
awkward to try to adjust.

Change-Id: I87b3d56de41f0ba5658c1240ddfc7ecf1c3c43af
---
 .zuul.yaml                                    | 346 +++++++++++++++++-
 playbooks/zuul/infra-puppet-apply/post.yaml   |   7 +
 playbooks/zuul/infra-puppet-apply/pre.yaml    |   4 +
 playbooks/zuul/infra-puppet-apply/run.yaml    |  12 +
 .../zuul/infra-puppet-beaker-rspec/run.yaml   |  54 +++
 playbooks/zuul/logstash-filters/post.yaml     |  15 +
 playbooks/zuul/logstash-filters/run.yaml      |  28 ++
 7 files changed, 464 insertions(+), 2 deletions(-)
 create mode 100644 playbooks/zuul/infra-puppet-apply/post.yaml
 create mode 100644 playbooks/zuul/infra-puppet-apply/pre.yaml
 create mode 100644 playbooks/zuul/infra-puppet-apply/run.yaml
 create mode 100644 playbooks/zuul/infra-puppet-beaker-rspec/run.yaml
 create mode 100644 playbooks/zuul/logstash-filters/post.yaml
 create mode 100644 playbooks/zuul/logstash-filters/run.yaml

diff --git a/.zuul.yaml b/.zuul.yaml
index 81dcfa92f3..8725710a74 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -1483,6 +1483,348 @@
           '/etc/apache2/': logs
           '/var/log/apache2/': logs
 
+- job:
+    name: system-config-puppet-apply-4-ubuntu-xenial
+    # Required to install z-c for puppet module installs.
+    parent: legacy-base
+    timeout: 1800
+    nodeset: ubuntu-xenial
+    vars:
+      puppet_version: 4
+    required-projects:
+      - opendev/system-config
+      - opendev/ansible-role-puppet
+      - opendev/puppet-accessbot
+      - opendev/puppet-ansible
+      - opendev/puppet-apparmor
+      - opendev/puppet-askbot
+      - opendev/puppet-asterisk
+      - opendev/puppet-bandersnatch
+      - opendev/puppet-bugdaystats
+      - opendev/puppet-bup
+      - opendev/puppet-cgit
+      - opendev/puppet-ciwatch
+      - opendev/puppet-diskimage_builder
+      - opendev/puppet-drupal
+      - opendev/puppet-elastic_recheck
+      - opendev/puppet-elasticsearch
+      - opendev/puppet-ethercalc
+      - opendev/puppet-etherpad_lite
+      - opendev/puppet-exim
+      - opendev/puppet-germqtt
+      - opendev/puppet-gerrit
+      - opendev/puppet-gerritbot
+      - opendev/puppet-github
+      - opendev/puppet-grafyaml
+      - opendev/puppet-graphite
+      - opendev/puppet-haveged
+      - opendev/puppet-hound
+      - opendev/puppet-httpd
+      - opendev/puppet-infracloud
+      - opendev/puppet-infra-cookiecutter
+      - opendev/puppet-ipsilon
+      - opendev/puppet-iptables
+      - opendev/puppet-jeepyb
+      - opendev/puppet-jenkins
+      - opendev/puppet-kerberos
+      - opendev/puppet-kibana
+      - opendev/puppet-lodgeit
+      - opendev/puppet-log_processor
+      - opendev/puppet-logrotate
+      - opendev/puppet-logstash
+      - opendev/puppet-lpmqtt
+      - opendev/puppet-mailman
+      - opendev/puppet-mediawiki
+      - opendev/puppet-meetbot
+      - opendev/puppet-mosquitto
+      - opendev/puppet-mqtt_statsd
+      - opendev/puppet-mysql_backup
+      - opendev/puppet-nodepool
+      - opendev/puppet-openafs
+      - opendev/puppet-openstackci
+      - opendev/puppet-openstack_health
+      - opendev/puppet-openstackid
+      - opendev/puppet-openstack_infra_spec_helper
+      - opendev/puppet-os_client_config
+      - opendev/puppet-packagekit
+      - opendev/puppet-pgsql_backup
+      - opendev/puppet-phabricator
+      - opendev/puppet-pip
+      - opendev/puppet-planet
+      - opendev/puppet-project_config
+      - opendev/puppet-ptgbot
+      - opendev/puppet-puppet
+      - opendev/puppet-redis
+      - opendev/puppet-refstack
+      - opendev/puppet-reviewday
+      - opendev/puppet-simpleproxy
+      - opendev/puppet-snmpd
+      - opendev/puppet-ssh
+      - opendev/puppet-ssl_cert_check
+      - opendev/puppet-statusbot
+      - opendev/puppet-storyboard
+      - opendev/puppet-subunit2sql
+      - opendev/puppet-sudoers
+      - opendev/puppet-tmpreaper
+      - opendev/puppet-translation_checksite
+      - opendev/puppet-ulimit
+      - opendev/puppet-unattended_upgrades
+      - opendev/puppet-unbound
+      - opendev/puppet-user
+      - opendev/puppet-vcsrepo
+      - opendev/puppet-zanata
+      - opendev/puppet-zuul
+    pre-run: playbooks/zuul/infra-puppet-apply/pre.yaml
+    run: playbooks/zuul/infra-puppet-apply/run.yaml
+    post-run: playbooks/zuul/infra-puppet-apply/post.yaml
+    irrelevant-files:
+      - docker/.*
+      - playbooks/.*
+
+- job:
+    name: system-config-puppet-beaker-rspec-infra
+    parent: legacy-base
+    description: |
+      Base job for beaker-rspec tests for Infra's puppet modules.
+    nodeset: ubuntu-xenial
+    run: playbooks/zuul/infra-puppet-beaker-rspec/run.yaml
+    vars:
+      nodeset: nodepool-xenial
+      project_src_dir: "{{ zuul.project.src_dir }}"
+    timeout: 3600
+    irrelevant-files:
+      - ^.*\.md$
+      - ^.*\.rst$
+      - ^doc/.*$
+      - ^etc/.*$
+      - ^metadata.json$
+      - ^releasenotes/.*$
+      - ^test-requirements.txt$
+    required-projects:
+      - openstack/project-config
+      - opendev/system-config
+      - opendev/puppet-openstack_infra_spec_helper
+      - opendev/puppet-bugdaystats
+      - opendev/puppet-mysql_backup
+      - opendev/puppet-openstackci
+      - opendev/puppet-zuul
+      - opendev/puppet-mqtt_statsd
+      - opendev/puppet-meetbot
+      - opendev/puppet-hound
+      - opendev/puppet-pip
+      - opendev/puppet-os_client_config
+      - opendev/puppet-openstackid
+      - opendev/puppet-bandersnatch
+      - opendev/puppet-project_config
+      - opendev/puppet-grafyaml
+      - opendev/puppet-refstack
+      - opendev/puppet-github
+      - opendev/puppet-ethercalc
+      - opendev/puppet-unattended_upgrades
+      - opendev/puppet-openafs
+      - opendev/puppet-httpd
+      - opendev/puppet-drupal
+      - opendev/puppet-subunit2sql
+      - opendev/puppet-reviewday
+      - opendev/puppet-kibana
+      - opendev/puppet-redis
+      - opendev/puppet-phabricator
+      - opendev/puppet-ssl_cert_check
+      - opendev/puppet-lpmqtt
+      - opendev/puppet-germqtt
+      - opendev/puppet-cgit
+      - opendev/puppet-packagekit
+      - opendev/puppet-haveged
+      - opendev/puppet-graphite
+      - opendev/puppet-diskimage_builder
+      - opendev/puppet-sudoers
+      - opendev/puppet-zanata
+      - opendev/puppet-logstash
+      - opendev/puppet-gerritbot
+      - opendev/puppet-asterisk
+      - opendev/puppet-statusbot
+      - opendev/puppet-gerrit
+      - opendev/puppet-mediawiki
+      - opendev/puppet-mailman
+      - opendev/puppet-exim
+      - opendev/puppet-tmpreaper
+      - opendev/puppet-elastic_recheck
+      - opendev/puppet-ulimit
+      - opendev/puppet-planet
+      - opendev/puppet-nodepool
+      - opendev/puppet-logrotate
+      - opendev/puppet-infracloud
+      - opendev/puppet-elasticsearch
+      - opendev/puppet-unbound
+      - opendev/puppet-storyboard
+      - opendev/puppet-openstack_health
+      - opendev/puppet-kerberos
+      - opendev/puppet-askbot
+      - opendev/puppet-log_processor
+      - opendev/puppet-simpleproxy
+      - opendev/puppet-iptables
+      - opendev/puppet-lodgeit
+      - opendev/puppet-etherpad_lite
+      - opendev/puppet-mosquitto
+      - opendev/puppet-bup
+      - opendev/puppet-pgsql_backup
+      - opendev/puppet-ansible
+      - opendev/puppet-ssh
+      - opendev/puppet-snmpd
+      - opendev/puppet-user
+      - opendev/puppet-jeepyb
+      - opendev/puppet-accessbot
+      - opendev/puppet-ptgbot
+      - opendev/puppet-jenkins
+      - opendev/puppet-vcsrepo
+
+- job:
+    name: system-config-puppet-beaker-rspec-puppet-4-infra
+    parent: system-config-puppet-beaker-rspec-infra
+    description: |
+      Run beaker-rspec functional tests with puppet 4 on Ubuntu Xenial.
+    vars:
+      puppet_version: 4
+
+- job:
+    name: system-config-puppet-beaker-rspec-puppet-4-centos-7-infra
+    parent: system-config-puppet-beaker-rspec-infra
+    nodeset: centos-7
+    description: |
+      Run beaker-rspec functional tests with puppet 4 on CentOS 7.
+    vars:
+      nodeset: nodepool-centos7
+      puppet_version: 4
+
+- job:
+    name: system-config-openstackci-beaker-puppet-4
+    parent: system-config-puppet-beaker-rspec-infra
+    description: |
+      Run beaker-rspec functional tests with puppet 4 on Ubuntu Xenial for the
+      puppet-openstackci module.
+    vars:
+      project_src_dir: "{{ zuul.projects['opendev.org/opendev/puppet-openstackci'].src_dir }}"
+      puppet_version: 4
+
+- project-template:
+    name: system-config-puppet-apply-jobs
+    description: |
+      Runs puppet apply tests for OpenStack Infrastructure.
+    check:
+      jobs:
+        - system-config-puppet-apply-4-ubuntu-xenial
+        - system-config-openstackci-beaker-puppet-4
+    gate:
+      jobs:
+        - system-config-puppet-apply-4-ubuntu-xenial
+        - system-config-openstackci-beaker-puppet-4
+
+- project-template:
+    name: system-config-puppet-beaker-jobs
+    description: |
+      Runs puppet beaker tests for OpenStack Infrastructure.
+      The jobs are run on Ubuntu Xenial.
+    check:
+      jobs:
+        - system-config-puppet-beaker-rspec-puppet-4-infra
+    gate:
+      jobs:
+        - system-config-puppet-beaker-rspec-puppet-4-infra
+
+- job:
+    name: system-config-legacy-logstash-filters
+    parent: legacy-base
+    required-projects:
+      - opendev/ansible-role-puppet
+      - openstack/logstash-filters
+      - opendev/puppet-accessbot
+      - opendev/puppet-ansible
+      - opendev/puppet-askbot
+      - opendev/puppet-asterisk
+      - opendev/puppet-bandersnatch
+      - opendev/puppet-bugdaystats
+      - opendev/puppet-bup
+      - opendev/puppet-cgit
+      - opendev/puppet-diskimage_builder
+      - opendev/puppet-drupal
+      - opendev/puppet-elastic_recheck
+      - opendev/puppet-elasticsearch
+      - opendev/puppet-ethercalc
+      - opendev/puppet-etherpad_lite
+      - opendev/puppet-exim
+      - opendev/puppet-germqtt
+      - opendev/puppet-gerrit
+      - opendev/puppet-gerritbot
+      - opendev/puppet-github
+      - opendev/puppet-grafyaml
+      - opendev/puppet-graphite
+      - opendev/puppet-haveged
+      - opendev/puppet-hound
+      - opendev/puppet-httpd
+      - opendev/puppet-infracloud
+      - opendev/puppet-iptables
+      - opendev/puppet-jeepyb
+      - opendev/puppet-jenkins
+      - opendev/puppet-kerberos
+      - opendev/puppet-kibana
+      - opendev/puppet-lodgeit
+      - opendev/puppet-log_processor
+      - opendev/puppet-logrotate
+      - opendev/puppet-logstash
+      - opendev/puppet-lpmqtt
+      - opendev/puppet-mailman
+      - opendev/puppet-mediawiki
+      - opendev/puppet-meetbot
+      - opendev/puppet-mosquitto
+      - opendev/puppet-mqtt_statsd
+      - opendev/puppet-mysql_backup
+      - opendev/puppet-nodepool
+      - opendev/puppet-openafs
+      - opendev/puppet-openstackci
+      - opendev/puppet-openstack_health
+      - opendev/puppet-openstackid
+      - opendev/puppet-os_client_config
+      - opendev/puppet-packagekit
+      - opendev/puppet-pgsql_backup
+      - opendev/puppet-phabricator
+      - opendev/puppet-pip
+      - opendev/puppet-planet
+      - opendev/puppet-project_config
+      - opendev/puppet-ptgbot
+      - opendev/puppet-redis
+      - opendev/puppet-refstack
+      - opendev/puppet-reviewday
+      - opendev/puppet-simpleproxy
+      - opendev/puppet-snmpd
+      - opendev/puppet-ssh
+      - opendev/puppet-ssl_cert_check
+      - opendev/puppet-statusbot
+      - opendev/puppet-storyboard
+      - opendev/puppet-subunit2sql
+      - opendev/puppet-sudoers
+      - opendev/puppet-tmpreaper
+      - opendev/puppet-ulimit
+      - opendev/puppet-unattended_upgrades
+      - opendev/puppet-unbound
+      - opendev/puppet-user
+      - opendev/puppet-vcsrepo
+      - opendev/puppet-zanata
+      - opendev/puppet-zuul
+      - opendev/system-config
+    run: playbooks/zuul/logstash-filters/run.yaml
+    post-run: playbooks/zuul/logstash-filters/post.yaml
+    timeout: 1800
+    nodeset: ubuntu-xenial
+
+- project-template:
+    name: system-config-logstash-filter-jobs
+    check:
+      jobs:
+        - system-config-legacy-logstash-filters
+    gate:
+      jobs:
+        - system-config-legacy-logstash-filters
+
 # Make sure only one run of a system-config playbook happens at a time
 - semaphore:
     name: infra-prod-playbook
@@ -2136,8 +2478,8 @@
     templates:
       - system-config-zuul-role-integration
       - system-config-gerrit-images
-      - infra-puppet-apply-jobs
-      - infra-logstash-filter-jobs
+      - system-config-puppet-apply-jobs
+      - system-config-logstash-filter-jobs
     check:
       jobs:
         - opendev-tox-docs
diff --git a/playbooks/zuul/infra-puppet-apply/post.yaml b/playbooks/zuul/infra-puppet-apply/post.yaml
new file mode 100644
index 0000000000..00b142dcf6
--- /dev/null
+++ b/playbooks/zuul/infra-puppet-apply/post.yaml
@@ -0,0 +1,7 @@
+- hosts: all
+  tasks:
+  - name: Copy puppet-apply test output to log server
+    synchronize:
+      src: "{{ zuul.projects['opendev.org/opendev/system-config'].src_dir }}/applytest/"
+      dest: '{{ zuul.executor.log_root }}/applytest/'
+      mode: pull
diff --git a/playbooks/zuul/infra-puppet-apply/pre.yaml b/playbooks/zuul/infra-puppet-apply/pre.yaml
new file mode 100644
index 0000000000..8a37515876
--- /dev/null
+++ b/playbooks/zuul/infra-puppet-apply/pre.yaml
@@ -0,0 +1,4 @@
+- hosts: all
+  roles:
+    - role: bindep
+      bindep_profile: test
diff --git a/playbooks/zuul/infra-puppet-apply/run.yaml b/playbooks/zuul/infra-puppet-apply/run.yaml
new file mode 100644
index 0000000000..874924dde4
--- /dev/null
+++ b/playbooks/zuul/infra-puppet-apply/run.yaml
@@ -0,0 +1,12 @@
+- hosts: all
+  name: infra-puppet-apply
+  tasks:
+
+    - name: Run puppet-apply test
+      shell: ./tools/apply-test.sh
+      args:
+        chdir: "{{ zuul.projects['opendev.org/opendev/system-config'].src_dir }}"
+      environment:
+        PUPPET_VERSION: "{{ puppet_version | default('4') }}"
+      tags:
+        - skip_ansible_lint
diff --git a/playbooks/zuul/infra-puppet-beaker-rspec/run.yaml b/playbooks/zuul/infra-puppet-beaker-rspec/run.yaml
new file mode 100644
index 0000000000..5d9adb0361
--- /dev/null
+++ b/playbooks/zuul/infra-puppet-beaker-rspec/run.yaml
@@ -0,0 +1,54 @@
+- hosts: all
+  name: Beaker-rspec functional testing for Infra puppet modules
+  roles:
+    - bindep
+
+  tasks:
+
+    - name: Set up SSH for beaker
+      shell:
+        cmd: |
+          echo "" | sudo tee -a /etc/ssh/sshd_config
+          echo "Match address 127.0.0.1" | sudo tee -a /etc/ssh/sshd_config
+          echo "    PermitRootLogin without-password" | sudo tee -a /etc/ssh/sshd_config
+          echo "" | sudo tee -a /etc/ssh/sshd_config
+          echo "Match address ::1" | sudo tee -a /etc/ssh/sshd_config
+          echo "    PermitRootLogin without-password" | sudo tee -a /etc/ssh/sshd_config
+          mkdir -p ~/.ssh
+          ssh-keygen -f ~/.ssh/id_rsa -b 2048 -P ""
+          sudo mkdir -p /root/.ssh
+          cat ~/.ssh/id_rsa.pub | sudo tee -a /root/.ssh/authorized_keys
+          if [ -f /usr/bin/yum ]; then
+              sudo systemctl reload sshd
+          elif [ -f /usr/bin/apt-get ]; then
+              sudo service ssh restart
+          fi
+
+    - name: Create local gems directory
+      file:
+        state: directory
+        path: '{{ ansible_user_dir }}/.bundled_gems'
+
+    - name: Install ruby dependencies
+      become: yes
+      shell:
+        cmd: |
+          if [ -f /usr/bin/yum ]; then
+              yum install ruby-devel gcc-c++ -y
+          elif [ -f /usr/bin/apt-get ]; then
+              apt-get install ruby-dev -y
+          fi
+
+    - name: Execute acceptance tests
+      shell:
+        cmd: |
+          gem install bundler --no-ri --no-rdoc --verbose --version '<2.0.0'
+          $GEM_HOME/bin/bundle install --retry 3
+          export BEAKER_set={{ nodeset }}
+          export BEAKER_debug=yes
+          export BEAKER_color=no
+          $GEM_HOME/bin/bundle exec rspec spec/acceptance
+        chdir: "{{ project_src_dir }}"
+      environment:
+        'GEM_HOME': "{{ ansible_user_dir }}/.bundled_gems"
+        'PUPPET_VERSION': "{{ puppet_version | default('4') }}"
diff --git a/playbooks/zuul/logstash-filters/post.yaml b/playbooks/zuul/logstash-filters/post.yaml
new file mode 100644
index 0000000000..2ac6b41d4c
--- /dev/null
+++ b/playbooks/zuul/logstash-filters/post.yaml
@@ -0,0 +1,15 @@
+- hosts: primary
+  tasks:
+
+    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
+      synchronize:
+        src: '{{ ansible_user_dir }}/workspace/'
+        dest: '{{ zuul.executor.log_root }}'
+        mode: pull
+        copy_links: true
+        verify_host: true
+        rsync_opts:
+          - --include=/opendev/system-config/applytest/**
+          - --include=*/
+          - --exclude=*
+          - --prune-empty-dirs
diff --git a/playbooks/zuul/logstash-filters/run.yaml b/playbooks/zuul/logstash-filters/run.yaml
new file mode 100644
index 0000000000..0f22ecac8d
--- /dev/null
+++ b/playbooks/zuul/logstash-filters/run.yaml
@@ -0,0 +1,28 @@
+- hosts: all
+  name: Autoconverted job legacy-logstash-filters from old job gate-logstash-filters-ubuntu-xenial
+  tasks:
+
+    - name: Ensure legacy workspace directory
+      file:
+        path: '{{ ansible_user_dir }}/workspace'
+        state: directory
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          /usr/zuul-env/bin/zuul-cloner --cache-dir /opt/git \
+            https://opendev.org opendev/system-config
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          cd opendev/system-config
+          ./tools/logstash-filter-test.sh
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'