Run all main target programs' tests

Run tests from all main target programs, for that a new
refstack_target_programs var is defined. The var will cause the
role to execute  'refstack-client test' command for every given
target program.
Collect whole .stestr dir in the zuul jobs as there will be
multiple output files (for every given target program).
Bump the default guideline to the latest one (2020.11).
The patch also fixes a few style mistakes/typos.

Change-Id: I82ba0bf26b994d4e21acabcb015d798081a33da2

.zuul.yaml

@@ -31,8 +31,7 @@
       zuul_copy_output:
         "{{ zuul.projects['opendev.org/osf/refstack-client'].src_dir }}/etc/accounts.yaml": logs
         "{{ zuul.projects['opendev.org/osf/refstack-client'].src_dir }}/etc/tempest.conf": logs
-        "{{ zuul.projects['opendev.org/osf/refstack-client'].src_dir }}/.tempest/.stestr/1.json": logs
+        "{{ zuul.projects['opendev.org/osf/refstack-client'].src_dir }}/.tempest/.stestr/": logs
-        "{{ zuul.projects['opendev.org/osf/refstack-client'].src_dir }}/.tempest/.stestr/1": logs
         "{{ zuul.projects['opendev.org/osf/refstack-client'].src_dir }}/.tempest/tempest.log": logs
     run: playbooks/refstack-client-devstack.yaml
     irrelevant-files:

README.md

@@ -26,9 +26,10 @@ deployment is working by passing refstack tests.
 | guideline                    | False    | 2020.06                                                             | String | Specific refstack guideline.                                                                            |
 | private_key_path_src *       | False    | None                                                                | String | If defined, the key defined by the param is copied to the targeted machine to private_key_path location.|
 | refstack_client_source       | False    | ~/.refstack-client                                                  | String | Destination where refstack-client will be cloned only if the dir doesn't exist or it's empty, otherwise it will be used as is.|
-|  tempestconf_source          | False    | None                                                                | String | A path to python-tempestconf's source. If not defined, refstack-client's default is applied.            |
+| tempestconf_source           | False    | None                                                                | String | A path to python-tempestconf's source. If not defined, refstack-client's default is applied.            |
 | refstack_client_version      | False    | HEAD                                                                | String | Version of refstack-client cloned from git.                                                             |
 | server                       | False    | https://refstack.openstack.org/api                                  | String | Server url where the results will be uploaded to if upload_results is True.                             |
+| refstack_target_programs     | False    | ['platform', 'compute', 'object']                                   | List | The tests within the specified target programs will be executed.                                          |
 | tempest_account_concurrency  | False    | 3                                                                   | Int    | A concurrency accounts.yaml file is generated with by tempest.                                          |
 | tempest_config_path          | False    | None                                                                | String | Destination of tempest configuration file to be used for running refstack tests.                        |
 | tempest_tag                  | False    | refstack-client's default                                           | String | Tempest will be cloned and checkouted to this specific tag.                                             |

defaults/main.yaml

@@ -16,7 +16,11 @@ download_artifacts: false
 dest_dir: "{{ lookup('env', 'PWD') }}"
 
 # the latest guideline by default
-guideline: "2020.06"
+guideline: "2020.11"
+refstack_target_programs:
+  - platform
+  - compute
+  - object
 url_cirros_image: "http://download.cirros-cloud.net/0.4.0/cirros-0.4.0-x86_64-disk.img"
 # a concurrency accounts.yaml file is generated with by tempest
 tempest_account_concurrency: 3

infrared_plugin/plugin.spec

@@ -80,7 +80,12 @@ subparsers:
             type: Value
             help: |
               A path or an URL to a test list text file containing specific
-              test cases.a
+              test cases.
+          refstack_target_programs:
+            type: ListValue
+            help: |
+              Tests within the specified target programs will be executed.
+              Usage: --refstack_target_programs platform,compute
           upload_results:
             type: Bool
             help: |

playbooks/refstack-client-devstack.yaml

@@ -24,7 +24,7 @@
       include_tasks: ../tasks/main.yaml
       vars:
         refstack_client_source: "/home/zuul/{{ zuul.projects['opendev.org/osf/refstack-client'].src_dir }}"
-        tempestconf_source: "/home/zuul/{{ zuul.projects['opendev.org/osf/python-tempestconf'].    src_dir }}"
+        tempestconf_source: "/home/zuul/{{ zuul.projects['opendev.org/osf/python-tempestconf'].src_dir }}"
         source_credentials: "{{ devstack_base_dir }}/devstack/openrc demo demo"
         source_admin_credentials: "{{ devstack_base_dir }}/devstack/openrc admin admin"
         additional_tempestconf_params: "auth.tempest_roles Member object-storage.operator_role Member"

tasks/main.yaml

@@ -27,8 +27,19 @@
 - include_tasks: post-tasks.yaml
   when: upload_results | bool or  download_artifacts | bool
 
-- name: Check if we passed refstack tests
+- when: test_list is defined
-  fail:
+  block:
-    msg: "Refstack tests failed."
+    - name: Check if we passed refstack tests
-  # the refstack_result var got set in run-refstack-client.yaml
+      fail:
-  when: refstack_result.rc > 0
+        msg: "Refstack tests failed."
+      # the refstack_result var got set in run-refstack-client.yaml
+      when: refstack_result.rc > 0
+
+- when: test_list is not defined
+  block:
+    - name: Check if we passed tests of all specified target programs
+      fail:
+        msg: "Refstack tests failed."
+      # the refstack_result var got set in run-refstack-client.yaml
+      when: item.rc > 0
+      with_items: "{{ refstack_result.results }}"

tasks/post-tasks.yaml

@@ -26,18 +26,19 @@
       shell: |
         set -ex
         source .venv/bin/activate
-        refstack-client upload -y "{{ ls_out.files[-1].path }}" \
+        refstack-client upload -y "{{ ls_out.files[-(item[0] | int + 1) | int].path }}" \
           --url {{ server }} \
           -i {{ private_key_path }}
       register: upload_out
       args:
         chdir: "{{ refstack_client_source }}"
         executable: /bin/bash
+      with_indexed_items: "{{ refstack_target_programs }}"
       when:
         - private_key_path is defined
 
     - debug:
-        msg: "{{ upload_out.stdout }}"
+        msg: "{{ upload_out }}"
 
 # this becomes handy when the role is executed on a remote node from
 # an executor node (e.g. a Jenkins job) when the below artifacts might be
@@ -46,19 +47,21 @@
   block:
     - name: Download results file in .json
       fetch:
-        src: "{{ ls_out.files[-1].path }}"
+        src: "{{ ls_out.files[-(item[0] | int + 1) | int].path }}"
         dest: "{{ dest_dir }}/test_results.json"
         flat: true
+      with_indexed_items: "{{ refstack_target_programs }}"
 
     - name: Download results file in subunit
       fetch:
-        src: "{{ ls_out.files[-1].path | splitext | first }}"
+        src: "{{ ls_out.files[-(item[0] | int + 1) | int].path | splitext | first }}"
         dest: "{{ dest_dir }}/test_results_subunit"
         flat: true
+      with_indexed_items: "{{ refstack_target_programs }}"
 
     - name: Dump output of upload command
       copy:
-        content: "{{ upload_out.stdout }}"
+        content: "{{ upload_out }}"
         dest: "{{ dest_dir }}/upload_output.txt"
         mode: '0644'
       delegate_to: localhost

tasks/run-refstack-client.yaml

@@ -32,7 +32,7 @@
   ignore_errors: true
   when: test_list is defined
 
-- name: Run tests associated with the specified guideline
+- name: "Run {{ refstack_target_programs }} program tests associated within the {{ guideline }} guideline"
   shell: |
     set -ex
     export PATH=$PATH:/usr/local/sbin:/usr/sbin
@@ -41,7 +41,7 @@
     refstack-client test \
       -c {{ path_to_tempest_config }} \
       -v \
-      --test-list "https://refstack.openstack.org/api/v1/guidelines/{{ guideline }}/tests?target=platform&type=required&alias=true&flag=false"
+      --test-list "https://refstack.openstack.org/api/v1/guidelines/{{ guideline }}/tests?target={{ item }}&type=required&alias=true&flag=false"
   args:
     chdir: "{{ refstack_client_source }}"
     executable: /bin/bash
@@ -50,4 +50,5 @@
   # the refstack_result is inspected in the last task of the role where it
   # fails if it failed here
   ignore_errors: true
+  with_items: "{{ refstack_target_programs }}"
   when: test_list is not defined