fetch-subunit-output: collect additional subunits (2nd try)

In addition to the main subunit file from zuul_work_dir,
collect the subunit files from the elements (directories)
of the zuul_additional_subunit_dirs list.

The default behavior is unchanged.

While the documentation of this role states that zuul_work_dir
contains an absolute path, this is not always true.
So make sure to not make any assumption about zuul_work_dir
in order to not fail spectacularly as it happened with the
previous patch[0].

Add also some tests for the role: both the basic case
and with an additional test directory.

[0] https://review.opendev.org/673885

Change-Id: Iabf2e0cf6d86e36a174778367186bbd39a65c3dd

roles/fetch-subunit-output/README.rst

@@ -7,6 +7,13 @@ Collect subunit outputs
 
    Directory to work in. It has to be a fully qualified path.
 
+.. zuul:rolevar:: zuul_additional_subunit_dirs
+   :default: []
+
+   List of additional directories which contains subunit files
+   to collect. The content of zuul_work_dir is always checked,
+   so it should not be added here.
+
 .. zuul:rolevar:: tox_envlist
 
    tox environment that was used to run the tests originally.

roles/fetch-subunit-output/defaults/main.yaml

@@ -1,3 +1,4 @@
 ---
 tox_envlist: ""
+zuul_additional_subunit_dirs: []
 zuul_work_dir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}"

roles/fetch-subunit-output/tasks/main.yaml

@@ -9,11 +9,37 @@
     - testr_command.rc == 0
     - testr_command.stdout_lines
   block:
+    - name: Get the list of directories with subunit files
+      set_fact:
+        all_subunit_dirs: "{{ [ zuul_work_dir ] + zuul_additional_subunit_dirs }}"
+
+    # The usage an independent target file instead of sending the output
+    # to zuul_work_dir prevents issues related to zuul_work_dir being
+    # a relative path, which may happen despite what the documentation
+    # of this role claims.
+    - name: Create a temporary file to store the subunit stream
+      tempfile:
+        state: file
+        prefix: subunit.
+      register: temp_subunit_file
 
     - name: Generate subunit file
       shell:
-        cmd: "{{ testr_command.stdout_lines[0] }} last --subunit > ./testrepository.subunit"
+        cmd: "{{ testr_command.stdout_lines[0] }} last --subunit >>{{ temp_subunit_file.path }}"
-        chdir: "{{ zuul_work_dir }}"
+        chdir: "{{ item }}"
+      loop: "{{ all_subunit_dirs }}"
+
+    - name: Copy the combined subunit file to the zuul work directory
+      copy:
+        src: "{{ temp_subunit_file.path }}"
+        dest: "{{ zuul_work_dir }}/testrepository.subunit"
+        remote_src: yes
+
+    - name: Remove the temporary file
+      file:
+        name: "{{ temp_subunit_file.path }}"
+        state: absent
+      ignore_errors: true
 
     - name: Process and fetch subunit results
       include: process.yaml

test-playbooks/base-roles/base.yaml

@@ -11,3 +11,4 @@
 - include: fetch-zuul-cloner.yaml
 - include: validate-host.yaml
 - include: fetch-output.yaml
+- include: fetch-subunit-output.yaml

test-playbooks/base-roles/fetch-subunit-output.yaml

@@ -0,0 +1,127 @@
+- name: Run the fetch-subunit-output role
+  hosts: all
+  vars:
+    tests_data:
+      main:
+        directory: "{{ zuul_work_dir }}"
+        test_pattern: "WorkingTest.test_success"
+      secondary:
+        directory: "/var/tmp/extratests"
+        test_pattern: "FailingTest.test_failure"
+  pre_tasks:
+    # Required packages; install them into a .tox path
+    # to cover the find-*.sh scripts in the role a bit more.
+    - name: Install stestr and subunit-output
+      pip:
+         name:
+           - stestr>=2.0.0,<2.6.0
+           - python-subunit
+         virtualenv: "{{ zuul_work_dir }}/.tox/utests/"
+
+    - name: Ensure that the test directories exists
+      file:
+        name: "{{ item.value.directory }}"
+        state: directory
+      loop: "{{ tests_data|dict2items }}"
+
+    - name: Copy the test files on all directories
+      copy:
+        src: "subunit_tests"
+        dest: "{{ item.value.directory }}"
+      loop: "{{ tests_data|dict2items }}"
+
+    - name: Prepare the test results on all directories
+      shell: |
+        . {{ zuul_work_dir }}/.tox/utests/bin/activate
+        stestr init
+        stestr run --test-path subunit_tests {{ item.value.test_pattern }}
+      args:
+        chdir: "{{ item.value.directory }}"
+      ignore_errors: yes
+      loop: "{{ tests_data|dict2items }}"
+  roles:
+    - role: fetch-subunit-output
+  post_tasks:
+    - name: Check that the testrepository file has been pulled
+      delegate_to: localhost
+      file:
+        path: "{{ zuul.executor.log_root }}/testrepository.subunit.gz"
+        state: file
+      register: local_subunit_file
+
+    - name: Check that HTML test result file has been pulled
+      delegate_to: localhost
+      file:
+        path: "{{ zuul.executor.log_root }}/testr_results.html.gz"
+        state: file
+      register: local_html_test_results
+
+    - name: Validate that files were pulled correctly
+      assert:
+        that:
+          - local_subunit_file is not changed
+          - local_subunit_file is succeeded
+          - local_html_test_results is not changed
+          - local_html_test_results is succeeded
+
+    # only one subunit file; the failed result should be hidden
+    - name: Check the content of the HTML file
+      delegate_to: localhost
+      shell: |
+        GLOBAL_RESULT=1
+        zgrep -q -E 'subunit_tests.test_working.WorkingTest.test_success$' \
+            {{ zuul.executor.log_root }}/testr_results.html.gz
+        T1=$?
+        zgrep -q -E 'subunit_tests.test_failing.FailingTest.test_failure.*_StringException:' \
+            {{ zuul.executor.log_root }}/testr_results.html.gz
+        T2=$?
+        if [ ${T1} -eq 0 ] && [ ${T2} -ne 0 ]; then
+            GLOBAL_RESULT=0
+        fi
+        exit $GLOBAL_RESULT
+
+
+# The following test(s) require(s) the previous playbook
+- name: Run the fetch-subunit-output role with multiple subunits
+  hosts: all
+  roles:
+    - role: fetch-subunit-output
+      zuul_additional_subunit_dirs:
+        - "/var/tmp/extratests"
+  post_tasks:
+    - name: Check that the testrepository file has been pulled
+      delegate_to: localhost
+      file:
+        path: "{{ zuul.executor.log_root }}/testrepository.subunit.gz"
+        state: file
+      register: local_subunit_file
+
+    - name: Check that HTML test result file has been pulled
+      delegate_to: localhost
+      file:
+        path: "{{ zuul.executor.log_root }}/testr_results.html.gz"
+        state: file
+      register: local_html_test_results
+
+    - name: Validate that files were pulled correctly
+      assert:
+        that:
+          - local_subunit_file is not changed
+          - local_subunit_file is succeeded
+          - local_html_test_results is not changed
+          - local_html_test_results is succeeded
+
+    - name: Check the content of the HTML file
+      delegate_to: localhost
+      shell: |
+        GLOBAL_RESULT=1
+        zgrep -q -E 'subunit_tests.test_working.WorkingTest.test_success$' \
+            {{ zuul.executor.log_root }}/testr_results.html.gz
+        T1=$?
+        zgrep -q -E 'subunit_tests.test_failing.FailingTest.test_failure.*_StringException:' \
+            {{ zuul.executor.log_root }}/testr_results.html.gz
+        T2=$?
+        if [ ${T1} -eq 0 ] && [ ${T2} -eq 0 ]; then
+            GLOBAL_RESULT=0
+        fi
+        exit $GLOBAL_RESULT

test-playbooks/base-roles/subunit_tests/test_failing.py

@@ -0,0 +1,21 @@
+# Copyright 2019 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+
+class FailingTest(unittest.TestCase):
+
+    def test_failure(self):
+        self.assertTrue(False)

test-playbooks/base-roles/subunit_tests/test_working.py

@@ -0,0 +1,21 @@
+# Copyright 2019 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+
+class WorkingTest(unittest.TestCase):
+
+    def test_success(self):
+        self.assertTrue(True)

zuul-tests.d/general-roles-jobs.yaml

@@ -117,6 +117,7 @@
       - ^roles/emit-job-header/.*
       - ^roles/ensure-output-dirs/.*
       - ^roles/fetch-output/.*
+      - ^roles/fetch-subunit-output/.*
       - ^roles/fetch-zuul-cloner/.*
       - ^roles/set-zuul-log-path-fact/.*
       - ^roles/prepare-workspace-git/.*