From e73a897642aa8fc3b6b8de39ed97223dc16c11c8 Mon Sep 17 00:00:00 2001 From: Andrey Kurilin Date: Thu, 7 Dec 2023 16:19:14 +0100 Subject: [PATCH] [ci] Include pytest html reports for functional job Change-Id: I35c68ab5ad3ff9ac2f12f383489a18d55d2d0bf1 --- .zuul.d/base.yaml | 2 + .zuul.d/docker-jobs.yaml | 2 + .zuul.d/rally-tox-functional.yaml | 5 ++ .../fetch-html-and-json-reports.yaml | 17 +++--- .../prepare-for-rally-task/tasks/main.yaml | 4 +- .../ci/playbooks/tox-functional-env-run.yaml | 58 +++++++++---------- tests/ci/pytest_launcher.py | 28 ++------- tox.ini | 2 + 8 files changed, 53 insertions(+), 65 deletions(-) diff --git a/.zuul.d/base.yaml b/.zuul.d/base.yaml index e4bd1ecb..d9a411e8 100644 --- a/.zuul.d/base.yaml +++ b/.zuul.d/base.yaml @@ -17,5 +17,7 @@ KEYSTONE_ADMIN_ENDPOINT: true devstack_plugins: rally-openstack: https://opendev.org/openstack/rally-openstack + rally_home_dir: '/opt/stack/.rally' + rally_results_dir: '/opt/stack/.rally/results' run: tests/ci/playbooks/run-rally-task.yaml post-run: tests/ci/playbooks/post-rally-task.yaml diff --git a/.zuul.d/docker-jobs.yaml b/.zuul.d/docker-jobs.yaml index 262bdd16..a0b2668b 100644 --- a/.zuul.d/docker-jobs.yaml +++ b/.zuul.d/docker-jobs.yaml @@ -40,6 +40,7 @@ docker_repository: "xrally/xrally-openstack" rally_plugin_name: "Dummy.openstack" rally_package_name: "rally-openstack" + rally_results_dir: "{{ zuul.project.src_dir }}/.test_results/" - job: name: rally-openstack-docker-build-and-push @@ -59,3 +60,4 @@ docker_repository: "xrally/xrally-openstack" rally_plugin_name: "Dummy.openstack" rally_package_name: "rally-openstack" + rally_results_dir: "{{ zuul.project.src_dir }}/.test_results/" diff --git a/.zuul.d/rally-tox-functional.yaml b/.zuul.d/rally-tox-functional.yaml index 96fd2455..8bd4a6d8 100644 --- a/.zuul.d/rally-tox-functional.yaml +++ b/.zuul.d/rally-tox-functional.yaml @@ -20,4 +20,9 @@ devstack_plugins: rally-openstack: https://opendev.org/openstack/rally-openstack tox_env: "functional" + rally_home_dir: '/opt/stack/.rally' + rally_results_dir: '/opt/stack/.rally/results/' + html_report: 'tox_func_report.html' + non_default_html_report: 'tox_func_report.html' run: tests/ci/playbooks/tox-functional-env-run.yaml + post-run: tests/ci/playbooks/fetch-html-and-json-reports.yaml diff --git a/tests/ci/playbooks/fetch-html-and-json-reports.yaml b/tests/ci/playbooks/fetch-html-and-json-reports.yaml index fe7074f8..51322154 100644 --- a/tests/ci/playbooks/fetch-html-and-json-reports.yaml +++ b/tests/ci/playbooks/fetch-html-and-json-reports.yaml @@ -1,10 +1,9 @@ - hosts: all vars: - results_dir: "{{ zuul.project.src_dir }}/.test_results/" - html_report: "{{ tox_env | default('self') }}_report.html" + html_report: "{% if non_default_html_report is defined %}{{ non_default_html_report }}{% else %}{{ tox_env | default('self') }}_report.html{% endif %}" json_report: "{{ tox_env | default('self') }}_report.json" tasks: - - shell: "ls {{ results_dir }}" + - shell: "ls {{ rally_results_dir }}" register: results_dir_stat ignore_errors: True @@ -12,7 +11,7 @@ become: yes when: results_dir_stat.rc == 0 synchronize: - src: "{{ results_dir }}" + src: "{{ rally_results_dir }}{% if not rally_results_dir.endswith('/')%}/{% endif %}" dest: '{{ zuul.executor.log_root }}/' mode: pull copy_links: true @@ -23,20 +22,20 @@ - --exclude=* - --prune-empty-dirs - - name: Return artifact to Zuul - when: html_report in results_dir_stat.stdout + - name: "Return artifact to Zuul - {{ html_report }}" zuul_return: data: zuul: artifacts: - name: "HTML report" url: "{{ html_report }}" + when: html_report in results_dir_stat.stdout - - name: Return artifact to Zuul - when: json_report in results_dir_stat.stdout + - name: "Return artifact to Zuul - {{ json_report}}" zuul_return: data: zuul: artifacts: - name: "JSON report" - url: "{{ json_report }}" \ No newline at end of file + url: "{{ json_report }}" + when: json_report in results_dir_stat.stdout diff --git a/tests/ci/playbooks/roles/prepare-for-rally-task/tasks/main.yaml b/tests/ci/playbooks/roles/prepare-for-rally-task/tasks/main.yaml index 1e027be8..23bc4e1b 100644 --- a/tests/ci/playbooks/roles/prepare-for-rally-task/tasks/main.yaml +++ b/tests/ci/playbooks/roles/prepare-for-rally-task/tasks/main.yaml @@ -11,7 +11,7 @@ become: True become_user: stack file: - path: '{{ rally_home_dir }}/results' + path: '{{ rally_results_dir }}' state: directory owner: stack group: stack @@ -20,7 +20,7 @@ become: True become_user: stack file: - path: '{{ rally_home_dir }}/results/{{ RALLY_OSPROFILER_CHART }}' + path: '{{ rally_results_dir }}/{{ RALLY_OSPROFILER_CHART }}' state: directory owner: stack group: stack diff --git a/tests/ci/playbooks/tox-functional-env-run.yaml b/tests/ci/playbooks/tox-functional-env-run.yaml index 609a1a12..2949c5bc 100644 --- a/tests/ci/playbooks/tox-functional-env-run.yaml +++ b/tests/ci/playbooks/tox-functional-env-run.yaml @@ -4,49 +4,45 @@ - hosts: controller vars: - rally_home_dir: '/opt/stack/.rally' rally_fake_image_path: '{{ rally_home_dir }}/extra/fake-image.img' rally_task_args_file: "100-percent-not-exist-file" # this task will not be launched, but we need to specify something real to # pass a check at 'prepare-for-rally-task' role. rally_task: "rally-jobs/simple-job.yaml" pip_install: "/opt/stack/rally-openstack/.tox/{{ tox_env }}/bin/pip install" + become: true + become_user: stack tasks: - name: "Trigger prepare tasks of rally-task-at-devstack zuul job" import_role: name: prepare-for-rally-task - - block: + - name: "Precreate a virtualenv for tox {{ tox_env }}" + command: tox -e {{ tox_env }} --notest + args: + chdir: "/opt/stack/rally-openstack" - - name: "Precreate a virtualenv for tox {{ tox_env }}" - command: tox -e {{ tox_env }} --notest - args: - chdir: /opt/stack/rally-openstack + - name: Check rally db connection + command: rally db show + register: rally_db_connection - - name: Check rally db connection - command: rally db show - register: rally_db_connection + # DevStack enables a special tool for MySQL performance counting. Since + # it is enabled globally, rally.conf includes it as well. + # tox -e functional utils reuse the same config file and fails + # if there is no this tool inside the venv + - name: Inject dbcounter plugin if needed + command: "{{ pip_install }} /opt/stack/devstack/tools/dbcounter" + when: "'plugin=dbcounter' in rally_db_connection.stdout" - # DevStack enables a special tool for MySQL performance counting. Since - # it is enabled globally, rally.conf includes it as well. - # tox -e functional utils reuse the same config file and fails - # if there is no this tool inside the venv - - name: Inject dbcounter plugin if needed - command: "{{ pip_install }} /opt/stack/devstack/tools/dbcounter" - when: "'plugin=dbcounter' in rally_db_connection.stdout" + - name: Install python lib for interacting with MySQL + command: "{{ pip_install }} PyMySQL>=0.7.6" + when: "'pymysql' in rally_db_connection.stdout" - - name: Install python lib for interacting with MySQL - command: "{{ pip_install }} PyMySQL>=0.7.6" - when: "'pymysql' in rally_db_connection.stdout" - - - name: Run tox command - shell: | - set -e - - export REQUESTS_CA_BUNDLE=/opt/stack/data/ca-bundle.pem - - tox -e {{ tox_env }} - args: - chdir: /opt/stack/rally-openstack - become: True - become_user: stack + - name: Run tox command + shell: tox -e {{ tox_env }} + environment: + REQUESTS_CA_BUNDLE: /opt/stack/data/ca-bundle.pem + REPORTS_ROOT: "{{ rally_results_dir }}/reports" + PYTEST_REPORT: "{{ rally_results_dir }}/{{ html_report }}" + args: + chdir: "/opt/stack/rally-openstack" diff --git a/tests/ci/pytest_launcher.py b/tests/ci/pytest_launcher.py index a293902b..f060f5ee 100755 --- a/tests/ci/pytest_launcher.py +++ b/tests/ci/pytest_launcher.py @@ -17,9 +17,10 @@ import subprocess import sys -PYTEST_REPORT = os.environ.get("PYTEST_REPORT", - ".test_results/pytest_results.html") -TESTR_REPORT = "testr_results.html" +TOX_ENV_NAME = os.environ["TOX_ENV_NAME"] +DEFAULT_REPORT_ROOT = os.environ.get("REPORTS_ROOT", ".test_results") +DEFAULT_REPORT = f"{DEFAULT_REPORT_ROOT}/{TOX_ENV_NAME}_report.html" +PYTEST_REPORT = os.environ.get("PYTEST_REPORT", DEFAULT_REPORT) PYTEST_ARGUMENTS = ("py.test" # base command " -vv" # show test names in logs " --html=%(html_report)s" # html report @@ -86,17 +87,7 @@ def main(args): print("Test(s) to launch (pytest format): %s" % path) - # NOTE(andreykurilin): we cannot publish pytest reports at gates, but we - # can mask them as testr reports. It looks like a dirty hack and I - # prefer to avoid it, but I see no other solutions at this point. - - # apply dirty hack only in gates. - if os.environ.get("ZUUL_PROJECT"): - pytest_report = TESTR_REPORT - else: - pytest_report = PYTEST_REPORT - - args = PYTEST_ARGUMENTS % {"html_report": pytest_report, + args = PYTEST_ARGUMENTS % {"html_report": PYTEST_REPORT, "path": path, "concurrency": args.concurrency or "auto"} try: @@ -104,15 +95,6 @@ def main(args): stderr=subprocess.STDOUT) except subprocess.CalledProcessError: # NOTE(andreykurilin): it is ok, since tests can fail. - exit_code = 1 - else: - exit_code = 0 - - if os.path.exists(pytest_report) and os.environ.get("ZUUL_PROJECT"): - subprocess.check_call(["gzip", "-9", "-f", pytest_report], - stderr=subprocess.STDOUT) - - if exit_code == 1: error("") diff --git a/tox.ini b/tox.ini index 98af474d..16a921c6 100644 --- a/tox.ini +++ b/tox.ini @@ -33,6 +33,8 @@ passenv = NO_PROXY REQUESTS_CA_BUNDLE HOME + REPORTS_ROOT + PYTEST_REPORT [testenv:pep8] deps = -r{toxinidir}/test-requirements.txt