openinfra/interop
Code Issues Proposed changes

Merge "Fix test discovery by checktest.py"

Browse Source
This commit is contained in:
Zuul 2021-09-15 11:21:19 +00:00 committed by Gerrit Code Review
commit 2de0e49b94
2 changed files with 60 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
tools/checktests.py
View File

@ -1,4 +1,5 @@
# Copyright 2018, OpenStack Foundation
# Copyright 2021, Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -13,12 +14,12 @@
# under the License.
import argparse
import ast
import importlib
import json
import os
import re
import sys
import unittest
def get_v1_version(guideline):
@ -90,7 +91,7 @@ def get_submodules(parent_module_name):
if not os.path.exists(os.path.join(root, '__init__.py')):
continue
for f in files:
if f.endswith('.py'):
if f.endswith('.py') and not f == "__init__.py":
module_path = root + '/' + f
module_name = root_name + '.' + os.path.splitext(f)[0]
if module_name not in submodules:
@ -98,26 +99,42 @@ def get_submodules(parent_module_name):
return submodules
def get_tests(module_name):
submodules = get_submodules(module_name)
tests = {}
for module_name in submodules:
filename = submodules[module_name]
with open(filename, 'r') as f:
source = f.read()
parsed = ast.parse(source)
for node in parsed.body:
if node.__class__ is ast.ClassDef:
for classnode in node.body:
if (classnode.__class__ is ast.FunctionDef and
classnode.name.startswith('test_')):
for decorator in classnode.decorator_list:
if hasattr(decorator, 'func'):
if decorator.func.attr == 'idempotent_id':
tests['id-' + decorator.args[0].s] = \
module_name + "." + node.name + "." + \
classnode.name
return tests
def get_module_tests(tests, parsed_test):
if isinstance(tests, unittest.TestCase):
test_description = tests.id()
test_uuid_regex = r'id-\w{8}-\w{4}-\w{4}-\w{4}-\w{12}'
test_id = re.search(test_uuid_regex, test_description)
if not test_id:
return parsed_test
test_id = test_id.group(0)
test_name = test_description.split("[")[0]
test_list = parsed_test.get(test_id, [])
test_list.append(test_name)
parsed_test[test_id] = test_list
return parsed_test
elif not isinstance(tests, unittest.suite.TestSuite):
return
for test in tests:
parsed_test = get_module_tests(test, parsed_test)
return parsed_test
def get_tests(submodules):
loader = unittest.TestLoader()
parsed_tests = {}
for submodule in submodules:
try:
tests = loader.loadTestsFromName(submodule)
parsed_tests = get_module_tests(tests, parsed_tests)
except Exception as e:
print("Unable to load: {}. Exception: {}".format(submodule, e))
return parsed_tests
def run():
@ -133,10 +150,10 @@ def run():
'against')
args = parser.parse_args()
guideline = load_guideline(args.guideline_file)
required = get_required_tests(guideline)
tests = get_tests(args.testlib)
submodules = get_submodules(args.testlib)
lib_tests = get_tests(submodules)
missing_uuids = []
missing_tests = {}
@ -144,10 +161,11 @@ def run():
for test in required:
uuid = test[0]
testnames = test[1]
if uuid not in tests:
if uuid not in lib_tests:
missing_uuids.append(test)
else:
if tests[uuid] not in testnames:
in_testnames = [test in lib_tests[uuid] for test in testnames]
if not any(in_testnames):
missing_tests[uuid] = test
exit_code = 0
@ -182,7 +200,7 @@ def run():
" idempotent_id:\n"
" %s\n"
" names: " % (args.testlib,
uuid, tests[uuid],
uuid, lib_tests[uuid],
args.guideline_file,
missing_tests[uuid][0]))
for testname in missing_tests[uuid][1]:

27
tools/consistency.sh
View File

@ -77,17 +77,21 @@ if [[ -z $SFSDIR ]]; then
git clone https://opendev.org/openstack/manila-tempest-plugin $SFSDIR
fi
pip install $TEMPESTDIR
pip install $DNSDIR
pip install $ORCHESTRATIONDIR
pip install $SFSDIR
export PYTHONPATH=$TEMPESTDIR:$DNSDIR:$ORCHESTRATIONDIR:$SFSDIR
python3 ./tools/checktests.py --guideline guidelines/next.json
exit_1=$?
python3 ./tools/checktests.py --guideline add-ons/guidelines/dns.next.json --testlib designate_tempest_plugin
exit_2=$?
# TODO(kopecmartin) In order to unblock gates, skip check of manila tempest plugin until the following bug is resolved:
# https://storyboard.openstack.org/#!/story/2009146
# python3 ./tools/checktests.py --guideline add-ons/guidelines/shared_file_system.next.json --testlib manila_tempest_tests
# exit_3=$?
# TODO(lpiwowar) The consistency check of designate_tempest_plugin is omitted until
# https://bugs.launchpad.net/designate/+bug/1943115 is fixed.
# python3 ./tools/checktests.py --guideline add-ons/guidelines/dns.next.json --testlib designate_tempest_plugin
# exit_2=$?
python3 ./tools/checktests.py --guideline add-ons/guidelines/shared_file_system.next.json --testlib manila_tempest_tests
exit_3=$?
# TODO(kopecmartin) consistency check of heat_tempest_plugin is omitted intentionally until we improve the
# checktests.py so that it detects ids of the heat_tempest_plugin.api tests which don't use decorator.idempotent_id
# call to track the id
@ -96,10 +100,10 @@ exit_2=$?
python3 ./tools/checktests.py --guideline current_guideline
exit_5=$?
python3 ./tools/checktests.py --guideline add-ons/dns_current_guideline --testlib designate_tempest_plugin
exit_6=$?
# python3 ./tools/checktests.py --guideline add-ons/shared_file_system_current_guideline --testlib manila_tempest_tests
# exit_7=$?
# python3 ./tools/checktests.py --guideline add-ons/dns_current_guideline --testlib designate_tempest_plugin
# exit_6=$?
python3 ./tools/checktests.py --guideline add-ons/shared_file_system_current_guideline --testlib manila_tempest_tests
exit_7=$?
# python3 ./tools/checktests.py --guideline add-ons/orchestration_current_guideline --testlib heat_tempest_plugin
# exit_8=$?
@ -111,6 +115,5 @@ if [[ "${CLEANTEMPEST}" ]]; then
rm -rf $SFSDIR
fi
#! (( $exit_1 || $exit_2 || $exit_3 || $exit_4 || $exit_5 || $exit_6 || $exit_7 || $exit_8 ))
! (( $exit_1 || $exit_2 || $exit_5 || $exit_6 ))
! (( $exit_1 || $exit_3 || $exit_5 || $exit_7 ))