Allow parallelization in gate runner

Change-Id: I393a22cfcaecb00d14d8ac643bd4b7ffbba03b12 Signed-off-by: Andrii Ostapenko <andrii.ostapenko@att.com>
2020-07-07 00:11:25 -05:00 · 2020-07-07 00:11:25 -05:00 · fdcbd03784
commit fdcbd03784
parent a6a9146de4
6 changed files with 121 additions and 43 deletions
--- a/playbooks/osh-infra-gate-runner.yaml
+++ b/playbooks/osh-infra-gate-runner.yaml
@ -27,12 +27,14 @@
      file:
        path: "/tmp/artifacts"
        state: directory
+
    - name: Run gate scripts
      include_role:
-        name: osh-run-script
+        name: "{{ ([item] | flatten | length == 1) | ternary('osh-run-script', 'osh-run-script-set') }}"
      vars:
-        gate_script_path: "{{ item }}"
-      with_items: "{{ gate_scripts }}"
+        workload: "{{ [item] | flatten }}"
+      loop: "{{ gate_scripts }}"
+
    - name: "Downloads artifacts to executor"
      synchronize:
        src: "/tmp/artifacts"
--- a/roles/osh-run-script-set/defaults/main.yaml
+++ b/roles/osh-run-script-set/defaults/main.yaml
@ -0,0 +1,18 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+osh_params:
+  container_distro_name: ubuntu
+  container_distro_version: xenial
+  # feature_gates:
+...
--- a/roles/osh-run-script-set/tasks/main.yaml
+++ b/roles/osh-run-script-set/tasks/main.yaml
@ -0,0 +1,55 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- block:
+    - name: "Run script set {{ workload }}"
+      shell: |
+        set -xe;
+        {{ gate_script_path }}
+      loop: "{{ workload }}"
+      loop_control:
+        loop_var: gate_script_path
+        pause: 5
+      args:
+        chdir: "{{ zuul.project.src_dir }}/{{ gate_scripts_relative_path }}"
+      environment:
+        zuul_site_mirror_fqdn: "{{ zuul_site_mirror_fqdn }}"
+        OSH_EXTRA_HELM_ARGS: "{{ zuul_osh_extra_helm_args_relative_path | default('') }}"
+        OSH_PATH: "{{ zuul_osh_relative_path | default('../openstack-helm/') }}"
+        OSH_INFRA_PATH: "{{ zuul_osh_infra_relative_path | default('../openstack-helm-infra/') }}"
+        OPENSTACK_RELEASE: "{{ osh_params.openstack_release | default('') }}"
+        CONTAINER_DISTRO_NAME: "{{ osh_params.container_distro_name | default('') }}"
+        CONTAINER_DISTRO_VERSION: "{{ osh_params.container_distro_version | default('') }}"
+        FEATURE_GATES: "{{ osh_params.feature_gates | default('') }}"
+      # NOTE(aostapenko) using bigger than async_status timeout due to async_status issue with
+      # not recognizing timed out jobs: https://github.com/ansible/ansible/issues/25637
+      async: 3600
+      poll: 0
+      register: async_results
+
+    - name: Wait for script set to finish
+      async_status:
+        jid: '{{ item.ansible_job_id }}'
+      register: jobs
+      until: jobs.finished
+      delay: 5
+      retries: 360
+      loop: "{{ async_results.results }}"
+
+  always:
+    - name: Print script set output
+      shell: |
+          # NOTE(aostapenko) safely retrieving items for the unlikely case if jobs timed out in async_status
+          echo 'STDOUT:\n{{ item.get("stdout") | regex_replace("\'", "") }}\nSTDERR:\n{{ item.get("stderr") | regex_replace("\'", "") }}'
+      loop: "{{ jobs.results }}"
+...
--- a/roles/osh-run-script/tasks/main.yaml
+++ b/roles/osh-run-script/tasks/main.yaml
@ -11,10 +11,12 @@
 # limitations under the License.

 ---
- name: "Run script {{ gate_script_path }}"
+- name: "Run script {{ workload[0] }}"
  shell: |
    set -xe;
    {{ gate_script_path }}
+  vars:
+    gate_script_path: "{{ workload[0] }}"
  args:
    chdir: "{{ zuul.project.src_dir }}/{{ gate_scripts_relative_path }}"
  environment:
--- a/yamllint.conf
+++ b/yamllint.conf
@ -16,7 +16,8 @@ rules:
  document-start: enable
  empty-lines: enable
  empty-values: disable
-  hyphens: enable
+  hyphens:
+    ignore: .yamllint/zuul.d/jobs.yaml
  indentation:
    spaces: 2
    indent-sequences: whatever
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@ -139,14 +139,14 @@
      gate_scripts:
        - ./tools/deployment/osh-infra-logging/000-install-packages.sh
        - ./tools/deployment/osh-infra-logging/005-deploy-k8s.sh
-        - ./tools/deployment/osh-infra-logging/010-ingress.sh
-        - ./tools/deployment/osh-infra-logging/020-ceph.sh
-        - ./tools/deployment/osh-infra-logging/025-ceph-ns-activate.sh
-        - ./tools/deployment/osh-infra-logging/030-radosgw-osh-infra.sh
-        - ./tools/deployment/osh-infra-logging/040-ldap.sh
-        - ./tools/deployment/osh-infra-logging/050-elasticsearch.sh
-        - ./tools/deployment/osh-infra-logging/060-fluentd.sh
-        - ./tools/deployment/osh-infra-logging/070-kibana.sh
+        - - ./tools/deployment/osh-infra-logging/010-ingress.sh
+          - ./tools/deployment/osh-infra-logging/020-ceph.sh
+        - - ./tools/deployment/osh-infra-logging/025-ceph-ns-activate.sh
+          - ./tools/deployment/osh-infra-logging/030-radosgw-osh-infra.sh
+          - ./tools/deployment/osh-infra-logging/040-ldap.sh
+          - ./tools/deployment/osh-infra-logging/050-elasticsearch.sh
+        - - ./tools/deployment/osh-infra-logging/060-fluentd.sh
+          - ./tools/deployment/osh-infra-logging/070-kibana.sh
        - ./tools/deployment/osh-infra-logging/600-kibana-selenium.sh || true

 - job:
@ -183,24 +183,24 @@
      gate_scripts:
        - ./tools/deployment/osh-infra-monitoring/000-install-packages.sh
        - ./tools/deployment/osh-infra-monitoring/005-deploy-k8s.sh
-        - ./tools/deployment/osh-infra-monitoring/010-deploy-docker-registry.sh
-        - ./tools/deployment/osh-infra-monitoring/020-ingress.sh
-        - ./tools/deployment/osh-infra-monitoring/030-nfs-provisioner.sh
-        - ./tools/deployment/osh-infra-monitoring/040-ldap.sh
-        - ./tools/deployment/osh-infra-monitoring/045-mariadb.sh
-        - ./tools/deployment/osh-infra-monitoring/050-prometheus.sh
-        - ./tools/deployment/osh-infra-monitoring/060-alertmanager.sh
-        - ./tools/deployment/osh-infra-monitoring/070-kube-state-metrics.sh
-        - ./tools/deployment/osh-infra-monitoring/075-node-problem-detector.sh
-        - ./tools/deployment/osh-infra-monitoring/080-node-exporter.sh
-        - ./tools/deployment/osh-infra-monitoring/090-process-exporter.sh
-        - ./tools/deployment/osh-infra-monitoring/100-openstack-exporter.sh
-        - ./tools/deployment/osh-infra-monitoring/110-grafana.sh
-        - ./tools/deployment/osh-infra-monitoring/120-nagios.sh
-        - ./tools/deployment/osh-infra-monitoring/130-postgresql.sh
-        - ./tools/deployment/osh-infra-monitoring/600-grafana-selenium.sh || true
-        - ./tools/deployment/osh-infra-monitoring/610-prometheus-selenium.sh || true
-        - ./tools/deployment/osh-infra-monitoring/620-nagios-selenium.sh || true
+        - - ./tools/deployment/osh-infra-monitoring/010-deploy-docker-registry.sh
+          - ./tools/deployment/osh-infra-monitoring/020-ingress.sh
+          - ./tools/deployment/osh-infra-monitoring/030-nfs-provisioner.sh
+          - ./tools/deployment/osh-infra-monitoring/040-ldap.sh
+          - ./tools/deployment/osh-infra-monitoring/045-mariadb.sh
+        - - ./tools/deployment/osh-infra-monitoring/050-prometheus.sh
+          - ./tools/deployment/osh-infra-monitoring/060-alertmanager.sh
+          - ./tools/deployment/osh-infra-monitoring/070-kube-state-metrics.sh
+          - ./tools/deployment/osh-infra-monitoring/075-node-problem-detector.sh
+          - ./tools/deployment/osh-infra-monitoring/080-node-exporter.sh
+        - - ./tools/deployment/osh-infra-monitoring/090-process-exporter.sh
+          - ./tools/deployment/osh-infra-monitoring/100-openstack-exporter.sh
+          - ./tools/deployment/osh-infra-monitoring/110-grafana.sh
+          - ./tools/deployment/osh-infra-monitoring/120-nagios.sh
+          - ./tools/deployment/osh-infra-monitoring/130-postgresql.sh
+        - - ./tools/deployment/osh-infra-monitoring/600-grafana-selenium.sh || true
+          - ./tools/deployment/osh-infra-monitoring/610-prometheus-selenium.sh || true
+          - ./tools/deployment/osh-infra-monitoring/620-nagios-selenium.sh || true

 - job:
    name: openstack-helm-infra-federated-monitoring
@ -394,20 +394,20 @@
        - ./tools/deployment/openstack-support/000-install-packages.sh
        - ./tools/deployment/openstack-support/005-deploy-k8s.sh
        - ./tools/deployment/openstack-support/007-namespace-config.sh
-        - ./tools/deployment/openstack-support/010-ingress.sh
-        - ./tools/deployment/openstack-support/020-ceph.sh
+        - - ./tools/deployment/openstack-support/010-ingress.sh
+          - ./tools/deployment/openstack-support/020-ceph.sh
        - ./tools/deployment/openstack-support/025-ceph-ns-activate.sh
-        - ./tools/deployment/openstack-support/030-rabbitmq.sh
-        - ./tools/deployment/openstack-support/040-memcached.sh
-        - ./tools/deployment/openstack-support/050-libvirt.sh
-        - ./tools/deployment/openstack-support/060-openvswitch.sh
-        - ./tools/deployment/openstack-support/070-mariadb.sh
-        - ./tools/deployment/openstack-support/080-setup-client.sh
+        - - ./tools/deployment/openstack-support/030-rabbitmq.sh
+          - ./tools/deployment/openstack-support/040-memcached.sh
+          - ./tools/deployment/openstack-support/050-libvirt.sh
+          - ./tools/deployment/openstack-support/060-openvswitch.sh
+          - ./tools/deployment/openstack-support/070-mariadb.sh
+          - ./tools/deployment/openstack-support/080-setup-client.sh
        - ./tools/deployment/openstack-support/090-keystone.sh
-        - ./tools/deployment/openstack-support/100-ceph-radosgateway.sh
-        - ./tools/deployment/openstack-support/110-openstack-exporter.sh
-        - ./tools/deployment/openstack-support/120-powerdns.sh
-        - ./tools/deployment/openstack-support/130-cinder.sh
+        - - ./tools/deployment/openstack-support/100-ceph-radosgateway.sh
+          - ./tools/deployment/openstack-support/110-openstack-exporter.sh
+          - ./tools/deployment/openstack-support/120-powerdns.sh
+          - ./tools/deployment/openstack-support/130-cinder.sh

 - job:
    name: openstack-helm-infra-five-ubuntu