From ab8f9fc4032716f972b75bce08e8f90e04595b05 Mon Sep 17 00:00:00 2001 From: Luigi Toscano Date: Thu, 17 Oct 2019 00:42:13 +0200 Subject: [PATCH] fetch-subunit-output: collect additional subunits (2nd try) In addition to the main subunit file from zuul_work_dir, collect the subunit files from the elements (directories) of the zuul_additional_subunit_dirs list. The default behavior is unchanged. While the documentation of this role states that zuul_work_dir contains an absolute path, this is not always true. So make sure to not make any assumption about zuul_work_dir in order to not fail spectacularly as it happened with the previous patch[0]. Add also some tests for the role: both the basic case and with an additional test directory. [0] https://review.opendev.org/673885 Change-Id: Iabf2e0cf6d86e36a174778367186bbd39a65c3dd --- roles/fetch-subunit-output/README.rst | 7 + roles/fetch-subunit-output/defaults/main.yaml | 1 + roles/fetch-subunit-output/tasks/main.yaml | 30 ++++- test-playbooks/base-roles/base.yaml | 1 + .../base-roles/fetch-subunit-output.yaml | 127 ++++++++++++++++++ .../base-roles/subunit_tests/__init__.py | 0 .../base-roles/subunit_tests/test_failing.py | 21 +++ .../base-roles/subunit_tests/test_working.py | 21 +++ zuul-tests.d/general-roles-jobs.yaml | 1 + 9 files changed, 207 insertions(+), 2 deletions(-) create mode 100644 test-playbooks/base-roles/fetch-subunit-output.yaml create mode 100644 test-playbooks/base-roles/subunit_tests/__init__.py create mode 100644 test-playbooks/base-roles/subunit_tests/test_failing.py create mode 100644 test-playbooks/base-roles/subunit_tests/test_working.py diff --git a/roles/fetch-subunit-output/README.rst b/roles/fetch-subunit-output/README.rst index 283b1d786..2540dbfb0 100644 --- a/roles/fetch-subunit-output/README.rst +++ b/roles/fetch-subunit-output/README.rst @@ -7,6 +7,13 @@ Collect subunit outputs Directory to work in. It has to be a fully qualified path. +.. zuul:rolevar:: zuul_additional_subunit_dirs + :default: [] + + List of additional directories which contains subunit files + to collect. The content of zuul_work_dir is always checked, + so it should not be added here. + .. zuul:rolevar:: tox_envlist tox environment that was used to run the tests originally. diff --git a/roles/fetch-subunit-output/defaults/main.yaml b/roles/fetch-subunit-output/defaults/main.yaml index 8cc97ff70..96c12e1dc 100644 --- a/roles/fetch-subunit-output/defaults/main.yaml +++ b/roles/fetch-subunit-output/defaults/main.yaml @@ -1,3 +1,4 @@ --- tox_envlist: "" +zuul_additional_subunit_dirs: [] zuul_work_dir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}" diff --git a/roles/fetch-subunit-output/tasks/main.yaml b/roles/fetch-subunit-output/tasks/main.yaml index 496ed7d43..887f568f3 100644 --- a/roles/fetch-subunit-output/tasks/main.yaml +++ b/roles/fetch-subunit-output/tasks/main.yaml @@ -9,11 +9,37 @@ - testr_command.rc == 0 - testr_command.stdout_lines block: + - name: Get the list of directories with subunit files + set_fact: + all_subunit_dirs: "{{ [ zuul_work_dir ] + zuul_additional_subunit_dirs }}" + + # The usage an independent target file instead of sending the output + # to zuul_work_dir prevents issues related to zuul_work_dir being + # a relative path, which may happen despite what the documentation + # of this role claims. + - name: Create a temporary file to store the subunit stream + tempfile: + state: file + prefix: subunit. + register: temp_subunit_file - name: Generate subunit file shell: - cmd: "{{ testr_command.stdout_lines[0] }} last --subunit > ./testrepository.subunit" - chdir: "{{ zuul_work_dir }}" + cmd: "{{ testr_command.stdout_lines[0] }} last --subunit >>{{ temp_subunit_file.path }}" + chdir: "{{ item }}" + loop: "{{ all_subunit_dirs }}" + + - name: Copy the combined subunit file to the zuul work directory + copy: + src: "{{ temp_subunit_file.path }}" + dest: "{{ zuul_work_dir }}/testrepository.subunit" + remote_src: yes + + - name: Remove the temporary file + file: + name: "{{ temp_subunit_file.path }}" + state: absent + ignore_errors: true - name: Process and fetch subunit results include: process.yaml diff --git a/test-playbooks/base-roles/base.yaml b/test-playbooks/base-roles/base.yaml index 346d84e0b..18ba3fc60 100644 --- a/test-playbooks/base-roles/base.yaml +++ b/test-playbooks/base-roles/base.yaml @@ -11,3 +11,4 @@ - include: fetch-zuul-cloner.yaml - include: validate-host.yaml - include: fetch-output.yaml +- include: fetch-subunit-output.yaml diff --git a/test-playbooks/base-roles/fetch-subunit-output.yaml b/test-playbooks/base-roles/fetch-subunit-output.yaml new file mode 100644 index 000000000..9bfc30197 --- /dev/null +++ b/test-playbooks/base-roles/fetch-subunit-output.yaml @@ -0,0 +1,127 @@ +- name: Run the fetch-subunit-output role + hosts: all + vars: + tests_data: + main: + directory: "{{ zuul_work_dir }}" + test_pattern: "WorkingTest.test_success" + secondary: + directory: "/var/tmp/extratests" + test_pattern: "FailingTest.test_failure" + pre_tasks: + # Required packages; install them into a .tox path + # to cover the find-*.sh scripts in the role a bit more. + - name: Install stestr and subunit-output + pip: + name: + - stestr>=2.0.0,<2.6.0 + - python-subunit + virtualenv: "{{ zuul_work_dir }}/.tox/utests/" + + - name: Ensure that the test directories exists + file: + name: "{{ item.value.directory }}" + state: directory + loop: "{{ tests_data|dict2items }}" + + - name: Copy the test files on all directories + copy: + src: "subunit_tests" + dest: "{{ item.value.directory }}" + loop: "{{ tests_data|dict2items }}" + + - name: Prepare the test results on all directories + shell: | + . {{ zuul_work_dir }}/.tox/utests/bin/activate + stestr init + stestr run --test-path subunit_tests {{ item.value.test_pattern }} + args: + chdir: "{{ item.value.directory }}" + ignore_errors: yes + loop: "{{ tests_data|dict2items }}" + roles: + - role: fetch-subunit-output + post_tasks: + - name: Check that the testrepository file has been pulled + delegate_to: localhost + file: + path: "{{ zuul.executor.log_root }}/testrepository.subunit.gz" + state: file + register: local_subunit_file + + - name: Check that HTML test result file has been pulled + delegate_to: localhost + file: + path: "{{ zuul.executor.log_root }}/testr_results.html.gz" + state: file + register: local_html_test_results + + - name: Validate that files were pulled correctly + assert: + that: + - local_subunit_file is not changed + - local_subunit_file is succeeded + - local_html_test_results is not changed + - local_html_test_results is succeeded + + # only one subunit file; the failed result should be hidden + - name: Check the content of the HTML file + delegate_to: localhost + shell: | + GLOBAL_RESULT=1 + zgrep -q -E 'subunit_tests.test_working.WorkingTest.test_success$' \ + {{ zuul.executor.log_root }}/testr_results.html.gz + T1=$? + zgrep -q -E 'subunit_tests.test_failing.FailingTest.test_failure.*_StringException:' \ + {{ zuul.executor.log_root }}/testr_results.html.gz + T2=$? + if [ ${T1} -eq 0 ] && [ ${T2} -ne 0 ]; then + GLOBAL_RESULT=0 + fi + exit $GLOBAL_RESULT + + +# The following test(s) require(s) the previous playbook +- name: Run the fetch-subunit-output role with multiple subunits + hosts: all + roles: + - role: fetch-subunit-output + zuul_additional_subunit_dirs: + - "/var/tmp/extratests" + post_tasks: + - name: Check that the testrepository file has been pulled + delegate_to: localhost + file: + path: "{{ zuul.executor.log_root }}/testrepository.subunit.gz" + state: file + register: local_subunit_file + + - name: Check that HTML test result file has been pulled + delegate_to: localhost + file: + path: "{{ zuul.executor.log_root }}/testr_results.html.gz" + state: file + register: local_html_test_results + + - name: Validate that files were pulled correctly + assert: + that: + - local_subunit_file is not changed + - local_subunit_file is succeeded + - local_html_test_results is not changed + - local_html_test_results is succeeded + + - name: Check the content of the HTML file + delegate_to: localhost + shell: | + GLOBAL_RESULT=1 + zgrep -q -E 'subunit_tests.test_working.WorkingTest.test_success$' \ + {{ zuul.executor.log_root }}/testr_results.html.gz + T1=$? + zgrep -q -E 'subunit_tests.test_failing.FailingTest.test_failure.*_StringException:' \ + {{ zuul.executor.log_root }}/testr_results.html.gz + T2=$? + if [ ${T1} -eq 0 ] && [ ${T2} -eq 0 ]; then + GLOBAL_RESULT=0 + fi + exit $GLOBAL_RESULT diff --git a/test-playbooks/base-roles/subunit_tests/__init__.py b/test-playbooks/base-roles/subunit_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test-playbooks/base-roles/subunit_tests/test_failing.py b/test-playbooks/base-roles/subunit_tests/test_failing.py new file mode 100644 index 000000000..d04a69402 --- /dev/null +++ b/test-playbooks/base-roles/subunit_tests/test_failing.py @@ -0,0 +1,21 @@ +# Copyright 2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import unittest + + +class FailingTest(unittest.TestCase): + + def test_failure(self): + self.assertTrue(False) diff --git a/test-playbooks/base-roles/subunit_tests/test_working.py b/test-playbooks/base-roles/subunit_tests/test_working.py new file mode 100644 index 000000000..d3657a7e4 --- /dev/null +++ b/test-playbooks/base-roles/subunit_tests/test_working.py @@ -0,0 +1,21 @@ +# Copyright 2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import unittest + + +class WorkingTest(unittest.TestCase): + + def test_success(self): + self.assertTrue(True) diff --git a/zuul-tests.d/general-roles-jobs.yaml b/zuul-tests.d/general-roles-jobs.yaml index ec7fb0ecf..e5f4008bf 100644 --- a/zuul-tests.d/general-roles-jobs.yaml +++ b/zuul-tests.d/general-roles-jobs.yaml @@ -117,6 +117,7 @@ - ^roles/emit-job-header/.* - ^roles/ensure-output-dirs/.* - ^roles/fetch-output/.* + - ^roles/fetch-subunit-output/.* - ^roles/fetch-zuul-cloner/.* - ^roles/set-zuul-log-path-fact/.* - ^roles/prepare-workspace-git/.*