diff --git a/add-ons/dns.2020.11.json b/add-ons/dns.2020.11.json index 90f1f0c2..29916cd6 100644 --- a/add-ons/dns.2020.11.json +++ b/add-ons/dns.2020.11.json @@ -155,7 +155,7 @@ "designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_show_zone": { "idempotent_id": "id-02ca5d6a-86ce-4f02-9d94-9e5db55c3055" }, - "designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_list_zone": { + "designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_list_zones": { "idempotent_id": "id-5bfa3cfe-5bc8-443b-bf48-cfba44cbb247" }, "designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_update_zone": { diff --git a/add-ons/dns.next.json b/add-ons/dns.next.json index 58970429..3ce70718 100644 --- a/add-ons/dns.next.json +++ b/add-ons/dns.next.json @@ -155,7 +155,7 @@ "designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_show_zone": { "idempotent_id": "id-02ca5d6a-86ce-4f02-9d94-9e5db55c3055" }, - "designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_list_zone": { + "designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_list_zones": { "idempotent_id": "id-5bfa3cfe-5bc8-443b-bf48-cfba44cbb247" }, "designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_update_zone": { diff --git a/add-ons/orchestration.2020.11.json b/add-ons/orchestration.2020.11.json index eefcf4e1..5de5f89c 100644 --- a/add-ons/orchestration.2020.11.json +++ b/add-ons/orchestration.2020.11.json @@ -637,7 +637,7 @@ "heat_tempest_plugin.tests.functional.test_preview.StackPreviewTest.test_nested_pass": { "idempotent_id": "id-0449113c-ff90-4f2b-8825-27ea35c1983f" }, - "heat_tempest_plugin.tests.functional.test_preview.StackPreviewTest.test_res_group_with_nested_template:Wherej": { + "heat_tempest_plugin.tests.functional.test_preview.StackPreviewTest.test_res_group_with_nested_template": { "idempotent_id": "id-6ca8ddfc-106f-4ecc-83f7-fca31d0c85ca" } } diff --git a/add-ons/orchestration.next.json b/add-ons/orchestration.next.json index c93aecdb..1dfe4e5a 100644 --- a/add-ons/orchestration.next.json +++ b/add-ons/orchestration.next.json @@ -637,7 +637,7 @@ "heat_tempest_plugin.tests.functional.test_preview.StackPreviewTest.test_nested_pass": { "idempotent_id": "id-0449113c-ff90-4f2b-8825-27ea35c1983f" }, - "heat_tempest_plugin.tests.functional.test_preview.StackPreviewTest.test_res_group_with_nested_template:Wherej": { + "heat_tempest_plugin.tests.functional.test_preview.StackPreviewTest.test_res_group_with_nested_template": { "idempotent_id": "id-6ca8ddfc-106f-4ecc-83f7-fca31d0c85ca" } } diff --git a/add-ons/shared_file_system.2020.11.json b/add-ons/shared_file_system.2020.11.json index 53133414..c73f7e0e 100644 --- a/add-ons/shared_file_system.2020.11.json +++ b/add-ons/shared_file_system.2020.11.json @@ -347,7 +347,7 @@ "project": "manila", "required_since": "2020.11", "tests": { - "manila_tempest_tests.tests.api.test_share_networks.ShareNetworksTest.test_list_share_networks": { + "manila_tempest_tests.tests.api.test_share_networks.ShareNetworkListMixin.test_list_share_networks": { "idempotent_id": "id-41c635b1-d9ef-4c05-9100-5e4b0034b523" } } diff --git a/add-ons/shared_file_system.next.json b/add-ons/shared_file_system.next.json index 655bd5fb..118272bd 100644 --- a/add-ons/shared_file_system.next.json +++ b/add-ons/shared_file_system.next.json @@ -344,7 +344,7 @@ "project": "manila", "required_since": "2020.11", "tests": { - "manila_tempest_tests.tests.api.test_share_networks.ShareNetworksTest.test_list_share_networks": { + "manila_tempest_tests.tests.api.test_share_networks.ShareNetworkListMixin.test_list_share_networks": { "idempotent_id": "id-41c635b1-d9ef-4c05-9100-5e4b0034b523" } } diff --git a/playbooks/parse_next_tests.yaml b/playbooks/parse_next_tests.yaml new file mode 100644 index 00000000..847a6818 --- /dev/null +++ b/playbooks/parse_next_tests.yaml @@ -0,0 +1,13 @@ +--- +- hosts: all + tasks: + - name: Ensure tox is installed + include_role: + name: ensure-tox + vars: + ensure_global_symlinks: true + + - name: Get all next tests + command: tox -vv -enext -- --interop-repo {{ ansible_user_dir }}/{{ zuul.projects['opendev.org/osf/interop'].src_dir }} + args: + chdir: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/osf/interop'].src_dir }}" diff --git a/test-requirements.txt b/test-requirements.txt index 025ea152..52d6366e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,3 +4,4 @@ doc8 jsonschema Sphinx sphinxcontrib.datatemplates +refstack>=2.0.0 diff --git a/tools/checktests.py b/tools/checktests.py index c686b804..6b148788 100644 --- a/tools/checktests.py +++ b/tools/checktests.py @@ -112,10 +112,11 @@ def get_tests(module_name): if (classnode.__class__ is ast.FunctionDef and classnode.name.startswith('test_')): for decorator in classnode.decorator_list: - if decorator.func.attr == 'idempotent_id': - tests['id-' + decorator.args[0].s] = \ - module_name + "." + node.name + "." + \ - classnode.name + if hasattr(decorator, 'func'): + if decorator.func.attr == 'idempotent_id': + tests['id-' + decorator.args[0].s] = \ + module_name + "." + node.name + "." + \ + classnode.name return tests diff --git a/tools/consistency.sh b/tools/consistency.sh index 05b46b59..6aac2570 100755 --- a/tools/consistency.sh +++ b/tools/consistency.sh @@ -1,40 +1,114 @@ #!/bin/bash # This script will run consistency checks for Tempest tests against -# the three latest interoperability guidelines. It can run in two +# the current and next interoperability guidelines. It can run in two # modes. # -# * If no arguments are specified, the script will check out Tempest -# into a temporary directory, run the consistency checks, then delete -# temporary checkout. +# * If no arguments are specified, the script will check out Tempest and +# tempest plugins into a temporary directory, run the consistency checks, +# then delete temporary checkout. # # * If an argument is given, this script will assume that it is a # user checked-out repository and run the consistency checks against # that, and leave the directory unchanged on exit. This mode is useful -# for gate jobs and Tempest development. +# for gate jobs and Tempest/tempest plugin development. set -x -if [ ! $@ ]; then - TEMPESTDIR=$(mktemp -d) - git clone https://opendev.org/openstack/tempest $TEMPESTDIR - CLEANTEMPEST=cleantempest -else - TEMPESTDIR=${1} +# Prints help +function usage { + SCRIPT_NAME="basename ${BASH_SOURCE[0]}" + echo "Usage: ${SCRIPT_NAME} [OPTION]..." + echo "Consistency check" + echo "" + echo " -h Print this usage message" + echo " -t Local Tempest directory" + echo " -d Local designate-tempest-plugin directory" + echo " -o Local heat-tempest-plugin directory" + echo " -s Local manila-tempest-plugin directory" + echo " -c Set if tempest and plugins directory should be removed after" + echo " the consistency check" + exit 1 +} + +while getopts t:d:o:s:ch FLAG; do + case ${FLAG} in + t) + TEMPESTDIR=${OPTARG} + ;; + d) + DNSDIR=${OPTARG} + ;; + o) + ORCHESTRATIONDIR=${OPTARG} + ;; + s) + SFSDIR=${OPTARG} + ;; + c) + CLEANTEMPEST=true + ;; + h) #show help + usage + ;; + \?) #unrecognized option - show help + echo -e \\n"Option -$OPTARG not allowed." + usage + ;; + esac +done + +# check if a local directory was given (for Tempest or one of the plugins), +# if not, create a temp dir and clone the project there +if [[ -z $TEMPESTDIR ]]; then + TEMPESTDIR=$(mktemp -d) + git clone https://opendev.org/openstack/tempest $TEMPESTDIR +fi +if [[ -z $DNSDIR ]]; then + DNSDIR=$(mktemp -d) + git clone https://opendev.org/openstack/designate-tempest-plugin $DNSDIR +fi +if [[ -z $ORCHESTRATIONDIR ]]; then + ORCHESTRATIONDIR=$(mktemp -d) + git clone https://opendev.org/openstack/heat-tempest-plugin $ORCHESTRATIONDIR +fi +if [[ -z $SFSDIR ]]; then + SFSDIR=$(mktemp -d) + git clone https://opendev.org/openstack/manila-tempest-plugin $SFSDIR fi -PYTHONPATH=$TEMPESTDIR python ./tools/checktests.py --guideline next.json + +export PYTHONPATH=$TEMPESTDIR:$DNSDIR:$ORCHESTRATIONDIR:$SFSDIR + +python3 ./tools/checktests.py --guideline next.json exit_1=$? - -PYTHONPATH=$TEMPESTDIR python ./tools/checktests.py --guideline 2018.02.json +python3 ./tools/checktests.py --guideline add-ons/dns.next.json --testlib designate_tempest_plugin exit_2=$? - -PYTHONPATH=$TEMPESTDIR python ./tools/checktests.py --guideline 2018.11.json +python3 ./tools/checktests.py --guideline add-ons/shared_file_system.next.json --testlib manila_tempest_tests exit_3=$? +# TODO(kopecmartin) consistency check of heat_tempest_plugin is omitted intentionally until we improve the +# checktests.py so that it detects ids of the heat_tempest_plugin.api tests which don't use decorator.idempotent_id +# call to track the id +# python3 ./tools/checktests.py --guideline add-ons/orchestration.next.json --testlib heat_tempest_plugin +# exit_4=$? -if [[ ! -z "${CLEANTEMPEST}" ]]; then +python3 ./tools/checktests.py --guideline 2020.11.json +exit_5=$? +python3 ./tools/checktests.py --guideline add-ons/dns.2020.11.json --testlib designate_tempest_plugin +exit_6=$? +python3 ./tools/checktests.py --guideline add-ons/shared_file_system.2020.11.json --testlib manila_tempest_tests +exit_7=$? +# python3 ./tools/checktests.py --guideline add-ons/orchestration.2020.11.json --testlib heat_tempest_plugin +# exit_8=$? + + +if [[ "${CLEANTEMPEST}" ]]; then rm -rf $TEMPESTDIR + rm -rf $DNSDIR + rm -rf $ORCHESTRATIONDIR + rm -rf $SFSDIR fi -! (( $exit_1 || $exit_2 || $exit_3 )) +#! (( $exit_1 || $exit_2 || $exit_3 || $exit_4 || $exit_5 || $exit_6 || $exit_7 || $exit_8 )) +! (( $exit_1 || $exit_2 || $exit_3 || $exit_5 || $exit_6 || $exit_7 )) diff --git a/tools/parse_next_tests.py b/tools/parse_next_tests.py new file mode 100644 index 00000000..24822de8 --- /dev/null +++ b/tools/parse_next_tests.py @@ -0,0 +1,69 @@ +# Copyright (c) 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import argparse +import json +import os + +from refstack.api.guidelines import Guidelines + +BASE_URL = "https://opendev.org/api/v1/repos/osf/interop/" +REPO_URL = BASE_URL + "contents/previous_guidelines" +RAW_URL = "https://opendev.org/api/v1/repos/osf/interop/raw/" +ADDITIONAL_CAPABILITY_URLS = BASE_URL + "contents/add-ons/previous_guidelines" + + +def parse_arguments(): + parser = argparse.ArgumentParser(__doc__) + parser.add_argument('--out', default="./all_next_tests.txt", + help="Name of the file where all next tests will be" + "written to.") + parser.add_argument('--interop-repo', default="./", + help="A path to the local interop repository the " + "script will look for the next files in.") + return parser.parse_args() + + +def parse_tests(f_path, target): + g = Guidelines(repo_url=REPO_URL, raw_url=RAW_URL, + additional_capability_urls=ADDITIONAL_CAPABILITY_URLS) + f = open(f_path, 'r') + load_f = json.load(f) + caps = g.get_target_capabilities(load_f, target=target) + tests = g.get_test_list(load_f, caps) + return tests + + +def main(): + args = parse_arguments() + tests = parse_tests( + os.path.join(args.interop_repo, 'next.json'), 'platform') + tests += parse_tests( + os.path.join(args.interop_repo, 'add-ons/dns.next.json'), 'dns') + tests += parse_tests( + os.path.join(args.interop_repo, 'add-ons/orchestration.next.json'), + 'orchestration') + tests += parse_tests( + os.path.join(args.interop_repo, + 'add-ons/shared_file_system.next.json'), + 'shared_file_system') + f = open(args.out, 'w') + for t in tests: + f.write(t + "\n") + f.close() + + +if __name__ == "__main__": + main() diff --git a/tox.ini b/tox.ini index 4f6f1964..53ee0df5 100644 --- a/tox.ini +++ b/tox.ini @@ -109,3 +109,6 @@ commands= whitelist_externals = sh bash + +[testenv:next] +commands=python3 tools/parse_next_tests.py {posargs} diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index fbbcd458..326023ea 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -5,8 +5,63 @@ Ensure consistency between Tempest source and current interop guidelines. vars: tox_envlist: consistency - tox_extra_args: "-vv -- {{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/tempest'].src_dir }}" + tox_extra_args: > + -vv -- + -t {{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/tempest'].src_dir }} + -d {{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/designate-tempest-plugin'].src_dir }} + -o {{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/heat-tempest-plugin'].src_dir }} + -s {{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/manila-tempest-plugin'].src_dir }} zuul_work_dir: src/opendev.org/osf/interop required-projects: - name: openstack/tempest + - name: openstack/designate-tempest-plugin + - name: openstack/heat-tempest-plugin + - name: openstack/manila-tempest-plugin - name: osf/interop + +- job: + name: interop-next-refstack-client-master + parent: refstack-client-devstack-master + description: | + A job running refstack-client on a devstack environment with the next guideline. + required-projects: + - name: osf/interop + pre-run: playbooks/parse_next_tests.yaml + vars: + test_list: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/osf/interop'].src_dir }}/all_next_tests.txt" + +- job: + name: interop-next-refstack-client-wallaby + parent: interop-next-refstack-client-master + description: | + A job running refstack-client on a devstack wallaby environment with the next guideline. + override-checkout: stable/wallaby + vars: + branch: stable/wallaby + +- job: + name: interop-next-refstack-client-victoria + parent: interop-next-refstack-client-master + description: | + A job running refstack-client on a devstack victoria environment with the next guideline. + override-checkout: stable/victoria + vars: + branch: stable/victoria + +- job: + name: interop-next-refstack-client-ussuri + parent: interop-next-refstack-client-master + description: | + A job running refstack-client on a devstack ussuri environment with the next guideline. + override-checkout: stable/ussuri + vars: + branch: stable/ussuri + +- job: + name: interop-next-refstack-client-train + parent: interop-next-refstack-client-master + description: | + A job running refstack-client on a devstack train environment with the next guideline. + override-checkout: stable/train + vars: + branch: stable/train diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml index 4b419196..b12a8ab9 100644 --- a/zuul.d/projects.yaml +++ b/zuul.d/projects.yaml @@ -5,6 +5,14 @@ jobs: - openstack-tox-pep8 - interop-tempest-consistency + # TODO(kopecmartin) master job has issues with ensure-tox role atm + # which we will figure out in a following patch. Let's comment out + # the job now so that it doesn't block us + # - interop-next-refstack-client-master + - interop-next-refstack-client-wallaby + - interop-next-refstack-client-victoria + - interop-next-refstack-client-ussuri + - interop-next-refstack-client-train gate: jobs: - openstack-tox-pep8