Allow parallelization in gate runner

Change-Id: I393a22cfcaecb00d14d8ac643bd4b7ffbba03b12
Signed-off-by: Andrii Ostapenko <andrii.ostapenko@att.com>

playbooks/osh-infra-gate-runner.yaml

@@ -27,12 +27,14 @@
       file:
         path: "/tmp/artifacts"
         state: directory
+
     - name: Run gate scripts
       include_role:
-        name: osh-run-script
+        name: "{{ ([item] | flatten | length == 1) | ternary('osh-run-script', 'osh-run-script-set') }}"
       vars:
-        gate_script_path: "{{ item }}"
+        workload: "{{ [item] | flatten }}"
-      with_items: "{{ gate_scripts }}"
+      loop: "{{ gate_scripts }}"
+
     - name: "Downloads artifacts to executor"
       synchronize:
         src: "/tmp/artifacts"

roles/osh-run-script-set/defaults/main.yaml

@@ -0,0 +1,18 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+osh_params:
+  container_distro_name: ubuntu
+  container_distro_version: xenial
+  # feature_gates:
+...

roles/osh-run-script-set/tasks/main.yaml

@@ -0,0 +1,55 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- block:
+    - name: "Run script set {{ workload }}"
+      shell: |
+        set -xe;
+        {{ gate_script_path }}
+      loop: "{{ workload }}"
+      loop_control:
+        loop_var: gate_script_path
+        pause: 5
+      args:
+        chdir: "{{ zuul.project.src_dir }}/{{ gate_scripts_relative_path }}"
+      environment:
+        zuul_site_mirror_fqdn: "{{ zuul_site_mirror_fqdn }}"
+        OSH_EXTRA_HELM_ARGS: "{{ zuul_osh_extra_helm_args_relative_path | default('') }}"
+        OSH_PATH: "{{ zuul_osh_relative_path | default('../openstack-helm/') }}"
+        OSH_INFRA_PATH: "{{ zuul_osh_infra_relative_path | default('../openstack-helm-infra/') }}"
+        OPENSTACK_RELEASE: "{{ osh_params.openstack_release | default('') }}"
+        CONTAINER_DISTRO_NAME: "{{ osh_params.container_distro_name | default('') }}"
+        CONTAINER_DISTRO_VERSION: "{{ osh_params.container_distro_version | default('') }}"
+        FEATURE_GATES: "{{ osh_params.feature_gates | default('') }}"
+      # NOTE(aostapenko) using bigger than async_status timeout due to async_status issue with
+      # not recognizing timed out jobs: https://github.com/ansible/ansible/issues/25637
+      async: 3600
+      poll: 0
+      register: async_results
+
+    - name: Wait for script set to finish
+      async_status:
+        jid: '{{ item.ansible_job_id }}'
+      register: jobs
+      until: jobs.finished
+      delay: 5
+      retries: 360
+      loop: "{{ async_results.results }}"
+
+  always:
+    - name: Print script set output
+      shell: |
+          # NOTE(aostapenko) safely retrieving items for the unlikely case if jobs timed out in async_status
+          echo 'STDOUT:\n{{ item.get("stdout") | regex_replace("\'", "") }}\nSTDERR:\n{{ item.get("stderr") | regex_replace("\'", "") }}'
+      loop: "{{ jobs.results }}"
+...

roles/osh-run-script/tasks/main.yaml

@@ -11,10 +11,12 @@
 # limitations under the License.
 
 ---
-- name: "Run script {{ gate_script_path }}"
+- name: "Run script {{ workload[0] }}"
   shell: |
     set -xe;
     {{ gate_script_path }}
+  vars:
+    gate_script_path: "{{ workload[0] }}"
   args:
     chdir: "{{ zuul.project.src_dir }}/{{ gate_scripts_relative_path }}"
   environment:

yamllint.conf

@@ -16,7 +16,8 @@ rules:
   document-start: enable
   empty-lines: enable
   empty-values: disable
-  hyphens: enable
+  hyphens:
+    ignore: .yamllint/zuul.d/jobs.yaml
   indentation:
     spaces: 2
     indent-sequences: whatever

zuul.d/jobs.yaml

@@ -139,13 +139,13 @@
       gate_scripts:
         - ./tools/deployment/osh-infra-logging/000-install-packages.sh
         - ./tools/deployment/osh-infra-logging/005-deploy-k8s.sh
-        - ./tools/deployment/osh-infra-logging/010-ingress.sh
+        - - ./tools/deployment/osh-infra-logging/010-ingress.sh
           - ./tools/deployment/osh-infra-logging/020-ceph.sh
-        - ./tools/deployment/osh-infra-logging/025-ceph-ns-activate.sh
+        - - ./tools/deployment/osh-infra-logging/025-ceph-ns-activate.sh
           - ./tools/deployment/osh-infra-logging/030-radosgw-osh-infra.sh
           - ./tools/deployment/osh-infra-logging/040-ldap.sh
           - ./tools/deployment/osh-infra-logging/050-elasticsearch.sh
-        - ./tools/deployment/osh-infra-logging/060-fluentd.sh
+        - - ./tools/deployment/osh-infra-logging/060-fluentd.sh
           - ./tools/deployment/osh-infra-logging/070-kibana.sh
         - ./tools/deployment/osh-infra-logging/600-kibana-selenium.sh || true
 
@@ -183,22 +183,22 @@
       gate_scripts:
         - ./tools/deployment/osh-infra-monitoring/000-install-packages.sh
         - ./tools/deployment/osh-infra-monitoring/005-deploy-k8s.sh
-        - ./tools/deployment/osh-infra-monitoring/010-deploy-docker-registry.sh
+        - - ./tools/deployment/osh-infra-monitoring/010-deploy-docker-registry.sh
           - ./tools/deployment/osh-infra-monitoring/020-ingress.sh
           - ./tools/deployment/osh-infra-monitoring/030-nfs-provisioner.sh
           - ./tools/deployment/osh-infra-monitoring/040-ldap.sh
           - ./tools/deployment/osh-infra-monitoring/045-mariadb.sh
-        - ./tools/deployment/osh-infra-monitoring/050-prometheus.sh
+        - - ./tools/deployment/osh-infra-monitoring/050-prometheus.sh
           - ./tools/deployment/osh-infra-monitoring/060-alertmanager.sh
           - ./tools/deployment/osh-infra-monitoring/070-kube-state-metrics.sh
           - ./tools/deployment/osh-infra-monitoring/075-node-problem-detector.sh
           - ./tools/deployment/osh-infra-monitoring/080-node-exporter.sh
-        - ./tools/deployment/osh-infra-monitoring/090-process-exporter.sh
+        - - ./tools/deployment/osh-infra-monitoring/090-process-exporter.sh
           - ./tools/deployment/osh-infra-monitoring/100-openstack-exporter.sh
           - ./tools/deployment/osh-infra-monitoring/110-grafana.sh
           - ./tools/deployment/osh-infra-monitoring/120-nagios.sh
           - ./tools/deployment/osh-infra-monitoring/130-postgresql.sh
-        - ./tools/deployment/osh-infra-monitoring/600-grafana-selenium.sh || true
+        - - ./tools/deployment/osh-infra-monitoring/600-grafana-selenium.sh || true
           - ./tools/deployment/osh-infra-monitoring/610-prometheus-selenium.sh || true
           - ./tools/deployment/osh-infra-monitoring/620-nagios-selenium.sh || true
 
@@ -394,17 +394,17 @@
         - ./tools/deployment/openstack-support/000-install-packages.sh
         - ./tools/deployment/openstack-support/005-deploy-k8s.sh
         - ./tools/deployment/openstack-support/007-namespace-config.sh
-        - ./tools/deployment/openstack-support/010-ingress.sh
+        - - ./tools/deployment/openstack-support/010-ingress.sh
           - ./tools/deployment/openstack-support/020-ceph.sh
         - ./tools/deployment/openstack-support/025-ceph-ns-activate.sh
-        - ./tools/deployment/openstack-support/030-rabbitmq.sh
+        - - ./tools/deployment/openstack-support/030-rabbitmq.sh
           - ./tools/deployment/openstack-support/040-memcached.sh
           - ./tools/deployment/openstack-support/050-libvirt.sh
           - ./tools/deployment/openstack-support/060-openvswitch.sh
           - ./tools/deployment/openstack-support/070-mariadb.sh
           - ./tools/deployment/openstack-support/080-setup-client.sh
         - ./tools/deployment/openstack-support/090-keystone.sh
-        - ./tools/deployment/openstack-support/100-ceph-radosgateway.sh
+        - - ./tools/deployment/openstack-support/100-ceph-radosgateway.sh
           - ./tools/deployment/openstack-support/110-openstack-exporter.sh
           - ./tools/deployment/openstack-support/120-powerdns.sh
           - ./tools/deployment/openstack-support/130-cinder.sh