Fix consistency job and guideline inconsistencies
Adds a job which runs all next tests on a devstack master and experimental jobs which run the same on supported stable releases. Fix names of a few manila_tempest_tests and designate_tempest_plugin tests so that they match the names from the plugins. Change-Id: Iedbaa09e425a29156304eea63b66a1ce69dd77bd
This commit is contained in:
Martin Kopec
parent
7cec1dc337
commit
a26948ee09
@ -155,7 +155,7 @@
|
||||
"designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_show_zone": {
|
||||
"idempotent_id": "id-02ca5d6a-86ce-4f02-9d94-9e5db55c3055"
|
||||
},
|
||||
"designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_list_zone": {
|
||||
"designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_list_zones": {
|
||||
"idempotent_id": "id-5bfa3cfe-5bc8-443b-bf48-cfba44cbb247"
|
||||
},
|
||||
"designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_update_zone": {
|
||||
|
||||
@ -155,7 +155,7 @@
|
||||
"designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_show_zone": {
|
||||
"idempotent_id": "id-02ca5d6a-86ce-4f02-9d94-9e5db55c3055"
|
||||
},
|
||||
"designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_list_zone": {
|
||||
"designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_list_zones": {
|
||||
"idempotent_id": "id-5bfa3cfe-5bc8-443b-bf48-cfba44cbb247"
|
||||
},
|
||||
"designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_update_zone": {
|
||||
|
||||
@ -637,7 +637,7 @@
|
||||
"heat_tempest_plugin.tests.functional.test_preview.StackPreviewTest.test_nested_pass": {
|
||||
"idempotent_id": "id-0449113c-ff90-4f2b-8825-27ea35c1983f"
|
||||
},
|
||||
"heat_tempest_plugin.tests.functional.test_preview.StackPreviewTest.test_res_group_with_nested_template:Wherej": {
|
||||
"heat_tempest_plugin.tests.functional.test_preview.StackPreviewTest.test_res_group_with_nested_template": {
|
||||
"idempotent_id": "id-6ca8ddfc-106f-4ecc-83f7-fca31d0c85ca"
|
||||
}
|
||||
}
|
||||
|
||||
@ -637,7 +637,7 @@
|
||||
"heat_tempest_plugin.tests.functional.test_preview.StackPreviewTest.test_nested_pass": {
|
||||
"idempotent_id": "id-0449113c-ff90-4f2b-8825-27ea35c1983f"
|
||||
},
|
||||
"heat_tempest_plugin.tests.functional.test_preview.StackPreviewTest.test_res_group_with_nested_template:Wherej": {
|
||||
"heat_tempest_plugin.tests.functional.test_preview.StackPreviewTest.test_res_group_with_nested_template": {
|
||||
"idempotent_id": "id-6ca8ddfc-106f-4ecc-83f7-fca31d0c85ca"
|
||||
}
|
||||
}
|
||||
|
||||
@ -347,7 +347,7 @@
|
||||
"project": "manila",
|
||||
"required_since": "2020.11",
|
||||
"tests": {
|
||||
"manila_tempest_tests.tests.api.test_share_networks.ShareNetworksTest.test_list_share_networks": {
|
||||
"manila_tempest_tests.tests.api.test_share_networks.ShareNetworkListMixin.test_list_share_networks": {
|
||||
"idempotent_id": "id-41c635b1-d9ef-4c05-9100-5e4b0034b523"
|
||||
}
|
||||
}
|
||||
|
||||
@ -344,7 +344,7 @@
|
||||
"project": "manila",
|
||||
"required_since": "2020.11",
|
||||
"tests": {
|
||||
"manila_tempest_tests.tests.api.test_share_networks.ShareNetworksTest.test_list_share_networks": {
|
||||
"manila_tempest_tests.tests.api.test_share_networks.ShareNetworkListMixin.test_list_share_networks": {
|
||||
"idempotent_id": "id-41c635b1-d9ef-4c05-9100-5e4b0034b523"
|
||||
}
|
||||
}
|
||||
|
||||
13
playbooks/parse_next_tests.yaml
Normal file
13
playbooks/parse_next_tests.yaml
Normal file
@ -0,0 +1,13 @@
|
||||
---
|
||||
- hosts: all
|
||||
tasks:
|
||||
- name: Ensure tox is installed
|
||||
include_role:
|
||||
name: ensure-tox
|
||||
vars:
|
||||
ensure_global_symlinks: true
|
||||
|
||||
- name: Get all next tests
|
||||
command: tox -vv -enext -- --interop-repo {{ ansible_user_dir }}/{{ zuul.projects['opendev.org/osf/interop'].src_dir }}
|
||||
args:
|
||||
chdir: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/osf/interop'].src_dir }}"
|
||||
@ -4,3 +4,4 @@ doc8
|
||||
jsonschema
|
||||
Sphinx
|
||||
sphinxcontrib.datatemplates
|
||||
refstack>=2.0.0
|
||||
|
||||
@ -112,10 +112,11 @@ def get_tests(module_name):
|
||||
if (classnode.__class__ is ast.FunctionDef and
|
||||
classnode.name.startswith('test_')):
|
||||
for decorator in classnode.decorator_list:
|
||||
if decorator.func.attr == 'idempotent_id':
|
||||
tests['id-' + decorator.args[0].s] = \
|
||||
module_name + "." + node.name + "." + \
|
||||
classnode.name
|
||||
if hasattr(decorator, 'func'):
|
||||
if decorator.func.attr == 'idempotent_id':
|
||||
tests['id-' + decorator.args[0].s] = \
|
||||
module_name + "." + node.name + "." + \
|
||||
classnode.name
|
||||
return tests
|
||||
|
||||
|
||||
|
||||
@ -1,40 +1,114 @@
|
||||
#!/bin/bash
|
||||
|
||||
# This script will run consistency checks for Tempest tests against
|
||||
# the three latest interoperability guidelines. It can run in two
|
||||
# the current and next interoperability guidelines. It can run in two
|
||||
# modes.
|
||||
#
|
||||
# * If no arguments are specified, the script will check out Tempest
|
||||
# into a temporary directory, run the consistency checks, then delete
|
||||
# temporary checkout.
|
||||
# * If no arguments are specified, the script will check out Tempest and
|
||||
# tempest plugins into a temporary directory, run the consistency checks,
|
||||
# then delete temporary checkout.
|
||||
#
|
||||
# * If an argument is given, this script will assume that it is a
|
||||
# user checked-out repository and run the consistency checks against
|
||||
# that, and leave the directory unchanged on exit. This mode is useful
|
||||
# for gate jobs and Tempest development.
|
||||
# for gate jobs and Tempest/tempest plugin development.
|
||||
|
||||
set -x
|
||||
|
||||
if [ ! $@ ]; then
|
||||
TEMPESTDIR=$(mktemp -d)
|
||||
git clone https://opendev.org/openstack/tempest $TEMPESTDIR
|
||||
CLEANTEMPEST=cleantempest
|
||||
else
|
||||
TEMPESTDIR=${1}
|
||||
# Prints help
|
||||
function usage {
|
||||
SCRIPT_NAME="basename ${BASH_SOURCE[0]}"
|
||||
echo "Usage: ${SCRIPT_NAME} [OPTION]..."
|
||||
echo "Consistency check"
|
||||
echo ""
|
||||
echo " -h Print this usage message"
|
||||
echo " -t Local Tempest directory"
|
||||
echo " -d Local designate-tempest-plugin directory"
|
||||
echo " -o Local heat-tempest-plugin directory"
|
||||
echo " -s Local manila-tempest-plugin directory"
|
||||
echo " -c Set if tempest and plugins directory should be removed after"
|
||||
echo " the consistency check"
|
||||
exit 1
|
||||
}
|
||||
|
||||
while getopts t:d:o:s:ch FLAG; do
|
||||
case ${FLAG} in
|
||||
t)
|
||||
TEMPESTDIR=${OPTARG}
|
||||
;;
|
||||
d)
|
||||
DNSDIR=${OPTARG}
|
||||
;;
|
||||
o)
|
||||
ORCHESTRATIONDIR=${OPTARG}
|
||||
;;
|
||||
s)
|
||||
SFSDIR=${OPTARG}
|
||||
;;
|
||||
c)
|
||||
CLEANTEMPEST=true
|
||||
;;
|
||||
h) #show help
|
||||
usage
|
||||
;;
|
||||
\?) #unrecognized option - show help
|
||||
echo -e \\n"Option -$OPTARG not allowed."
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# check if a local directory was given (for Tempest or one of the plugins),
|
||||
# if not, create a temp dir and clone the project there
|
||||
if [[ -z $TEMPESTDIR ]]; then
|
||||
TEMPESTDIR=$(mktemp -d)
|
||||
git clone https://opendev.org/openstack/tempest $TEMPESTDIR
|
||||
fi
|
||||
if [[ -z $DNSDIR ]]; then
|
||||
DNSDIR=$(mktemp -d)
|
||||
git clone https://opendev.org/openstack/designate-tempest-plugin $DNSDIR
|
||||
fi
|
||||
if [[ -z $ORCHESTRATIONDIR ]]; then
|
||||
ORCHESTRATIONDIR=$(mktemp -d)
|
||||
git clone https://opendev.org/openstack/heat-tempest-plugin $ORCHESTRATIONDIR
|
||||
fi
|
||||
if [[ -z $SFSDIR ]]; then
|
||||
SFSDIR=$(mktemp -d)
|
||||
git clone https://opendev.org/openstack/manila-tempest-plugin $SFSDIR
|
||||
fi
|
||||
|
||||
PYTHONPATH=$TEMPESTDIR python ./tools/checktests.py --guideline next.json
|
||||
|
||||
export PYTHONPATH=$TEMPESTDIR:$DNSDIR:$ORCHESTRATIONDIR:$SFSDIR
|
||||
|
||||
python3 ./tools/checktests.py --guideline next.json
|
||||
exit_1=$?
|
||||
|
||||
PYTHONPATH=$TEMPESTDIR python ./tools/checktests.py --guideline 2018.02.json
|
||||
python3 ./tools/checktests.py --guideline add-ons/dns.next.json --testlib designate_tempest_plugin
|
||||
exit_2=$?
|
||||
|
||||
PYTHONPATH=$TEMPESTDIR python ./tools/checktests.py --guideline 2018.11.json
|
||||
python3 ./tools/checktests.py --guideline add-ons/shared_file_system.next.json --testlib manila_tempest_tests
|
||||
exit_3=$?
|
||||
# TODO(kopecmartin) consistency check of heat_tempest_plugin is omitted intentionally until we improve the
|
||||
# checktests.py so that it detects ids of the heat_tempest_plugin.api tests which don't use decorator.idempotent_id
|
||||
# call to track the id
|
||||
# python3 ./tools/checktests.py --guideline add-ons/orchestration.next.json --testlib heat_tempest_plugin
|
||||
# exit_4=$?
|
||||
|
||||
if [[ ! -z "${CLEANTEMPEST}" ]]; then
|
||||
python3 ./tools/checktests.py --guideline 2020.11.json
|
||||
exit_5=$?
|
||||
python3 ./tools/checktests.py --guideline add-ons/dns.2020.11.json --testlib designate_tempest_plugin
|
||||
exit_6=$?
|
||||
python3 ./tools/checktests.py --guideline add-ons/shared_file_system.2020.11.json --testlib manila_tempest_tests
|
||||
exit_7=$?
|
||||
# python3 ./tools/checktests.py --guideline add-ons/orchestration.2020.11.json --testlib heat_tempest_plugin
|
||||
# exit_8=$?
|
||||
|
||||
|
||||
if [[ "${CLEANTEMPEST}" ]]; then
|
||||
rm -rf $TEMPESTDIR
|
||||
rm -rf $DNSDIR
|
||||
rm -rf $ORCHESTRATIONDIR
|
||||
rm -rf $SFSDIR
|
||||
fi
|
||||
|
||||
|
||||
! (( $exit_1 || $exit_2 || $exit_3 ))
|
||||
#! (( $exit_1 || $exit_2 || $exit_3 || $exit_4 || $exit_5 || $exit_6 || $exit_7 || $exit_8 ))
|
||||
! (( $exit_1 || $exit_2 || $exit_3 || $exit_5 || $exit_6 || $exit_7 ))
|
||||
|
||||
69
tools/parse_next_tests.py
Normal file
69
tools/parse_next_tests.py
Normal file
@ -0,0 +1,69 @@
|
||||
# Copyright (c) 2021 Red Hat, Inc.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
|
||||
from refstack.api.guidelines import Guidelines
|
||||
|
||||
BASE_URL = "https://opendev.org/api/v1/repos/osf/interop/"
|
||||
REPO_URL = BASE_URL + "contents/previous_guidelines"
|
||||
RAW_URL = "https://opendev.org/api/v1/repos/osf/interop/raw/"
|
||||
ADDITIONAL_CAPABILITY_URLS = BASE_URL + "contents/add-ons/previous_guidelines"
|
||||
|
||||
|
||||
def parse_arguments():
|
||||
parser = argparse.ArgumentParser(__doc__)
|
||||
parser.add_argument('--out', default="./all_next_tests.txt",
|
||||
help="Name of the file where all next tests will be"
|
||||
"written to.")
|
||||
parser.add_argument('--interop-repo', default="./",
|
||||
help="A path to the local interop repository the "
|
||||
"script will look for the next files in.")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def parse_tests(f_path, target):
|
||||
g = Guidelines(repo_url=REPO_URL, raw_url=RAW_URL,
|
||||
additional_capability_urls=ADDITIONAL_CAPABILITY_URLS)
|
||||
f = open(f_path, 'r')
|
||||
load_f = json.load(f)
|
||||
caps = g.get_target_capabilities(load_f, target=target)
|
||||
tests = g.get_test_list(load_f, caps)
|
||||
return tests
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_arguments()
|
||||
tests = parse_tests(
|
||||
os.path.join(args.interop_repo, 'next.json'), 'platform')
|
||||
tests += parse_tests(
|
||||
os.path.join(args.interop_repo, 'add-ons/dns.next.json'), 'dns')
|
||||
tests += parse_tests(
|
||||
os.path.join(args.interop_repo, 'add-ons/orchestration.next.json'),
|
||||
'orchestration')
|
||||
tests += parse_tests(
|
||||
os.path.join(args.interop_repo,
|
||||
'add-ons/shared_file_system.next.json'),
|
||||
'shared_file_system')
|
||||
f = open(args.out, 'w')
|
||||
for t in tests:
|
||||
f.write(t + "\n")
|
||||
f.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
3
tox.ini
3
tox.ini
@ -109,3 +109,6 @@ commands=
|
||||
whitelist_externals =
|
||||
sh
|
||||
bash
|
||||
|
||||
[testenv:next]
|
||||
commands=python3 tools/parse_next_tests.py {posargs}
|
||||
|
||||
@ -5,8 +5,63 @@
|
||||
Ensure consistency between Tempest source and current interop guidelines.
|
||||
vars:
|
||||
tox_envlist: consistency
|
||||
tox_extra_args: "-vv -- {{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/tempest'].src_dir }}"
|
||||
tox_extra_args: >
|
||||
-vv --
|
||||
-t {{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/tempest'].src_dir }}
|
||||
-d {{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/designate-tempest-plugin'].src_dir }}
|
||||
-o {{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/heat-tempest-plugin'].src_dir }}
|
||||
-s {{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/manila-tempest-plugin'].src_dir }}
|
||||
zuul_work_dir: src/opendev.org/osf/interop
|
||||
required-projects:
|
||||
- name: openstack/tempest
|
||||
- name: openstack/designate-tempest-plugin
|
||||
- name: openstack/heat-tempest-plugin
|
||||
- name: openstack/manila-tempest-plugin
|
||||
- name: osf/interop
|
||||
|
||||
- job:
|
||||
name: interop-next-refstack-client-master
|
||||
parent: refstack-client-devstack-master
|
||||
description: |
|
||||
A job running refstack-client on a devstack environment with the next guideline.
|
||||
required-projects:
|
||||
- name: osf/interop
|
||||
pre-run: playbooks/parse_next_tests.yaml
|
||||
vars:
|
||||
test_list: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/osf/interop'].src_dir }}/all_next_tests.txt"
|
||||
|
||||
- job:
|
||||
name: interop-next-refstack-client-wallaby
|
||||
parent: interop-next-refstack-client-master
|
||||
description: |
|
||||
A job running refstack-client on a devstack wallaby environment with the next guideline.
|
||||
override-checkout: stable/wallaby
|
||||
vars:
|
||||
branch: stable/wallaby
|
||||
|
||||
- job:
|
||||
name: interop-next-refstack-client-victoria
|
||||
parent: interop-next-refstack-client-master
|
||||
description: |
|
||||
A job running refstack-client on a devstack victoria environment with the next guideline.
|
||||
override-checkout: stable/victoria
|
||||
vars:
|
||||
branch: stable/victoria
|
||||
|
||||
- job:
|
||||
name: interop-next-refstack-client-ussuri
|
||||
parent: interop-next-refstack-client-master
|
||||
description: |
|
||||
A job running refstack-client on a devstack ussuri environment with the next guideline.
|
||||
override-checkout: stable/ussuri
|
||||
vars:
|
||||
branch: stable/ussuri
|
||||
|
||||
- job:
|
||||
name: interop-next-refstack-client-train
|
||||
parent: interop-next-refstack-client-master
|
||||
description: |
|
||||
A job running refstack-client on a devstack train environment with the next guideline.
|
||||
override-checkout: stable/train
|
||||
vars:
|
||||
branch: stable/train
|
||||
|
||||
@ -5,6 +5,14 @@
|
||||
jobs:
|
||||
- openstack-tox-pep8
|
||||
- interop-tempest-consistency
|
||||
# TODO(kopecmartin) master job has issues with ensure-tox role atm
|
||||
# which we will figure out in a following patch. Let's comment out
|
||||
# the job now so that it doesn't block us
|
||||
# - interop-next-refstack-client-master
|
||||
- interop-next-refstack-client-wallaby
|
||||
- interop-next-refstack-client-victoria
|
||||
- interop-next-refstack-client-ussuri
|
||||
- interop-next-refstack-client-train
|
||||
gate:
|
||||
jobs:
|
||||
- openstack-tox-pep8
|
||||
|
||||
Loading…
Reference in New Issue
Block a user