# Copyright 2018, OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. from __future__ import print_function import ast import argparse import importlib import json import os import re import sys def get_v1_version(guideline): schema = guideline.get('schema') if schema: v1regex = re.compile("^1.*") if v1regex.match(schema): return "1" return None def get_v2_version(guideline): metadata = guideline.get('metadata') if metadata and isinstance(metadata, dict): schema = metadata.get('schema') if schema: v2regex = re.compile("^2.*") if v2regex.match(schema): return "2" return None def get_guideline_version(guideline): return get_v1_version(guideline) or get_v2_version(guideline) def get_required_tests_v1(guideline): capabilities = guideline.get('capabilities') tests = [] for capability_name in capabilities: capability = capabilities.get(capability_name) for test_name in capability.get('tests'): test = capability.get('tests').get(test_name) if not test.get('flagged'): idempotent_id = test.get('idempotent_id') test_set = {test_name} if test.get('aliases'): test_set = test_set.union(set(test.get('aliases'))) tests.append((idempotent_id, test_set)) return tests def get_required_tests_v2(guideline): return get_required_tests_v1(guideline) def get_required_tests(guideline): if get_v1_version(guideline): return get_required_tests_v1(guideline) elif get_v2_version(guideline): return get_required_tests_v2(guideline) return None def load_guideline(guideline_file): with open(guideline_file) as f: guideline = json.load(f) return guideline def get_submodules(parent_module_name): module = importlib.import_module(parent_module_name) base_path = os.path.abspath(os.path.dirname(module.__file__)) base_index = len(base_path.split('/')) - 1 submodules = {} for root, dirnames, files in os.walk(base_path): root_name = '.'.join(root.split('/')[base_index:]) if not os.path.exists(os.path.join(root, '__init__.py')): continue for f in files: if f.endswith('.py'): module_path = root + '/' + f module_name = root_name + '.' + os.path.splitext(f)[0] if module_name not in submodules: submodules[module_name] = module_path return submodules def get_tests(module_name): submodules = get_submodules(module_name) tests = {} for module_name in submodules: filename = submodules[module_name] with open(filename, 'r') as f: source = f.read() parsed = ast.parse(source) for node in parsed.body: if node.__class__ is ast.ClassDef: for classnode in node.body: if (classnode.__class__ is ast.FunctionDef and classnode.name.startswith('test_')): for decorator in classnode.decorator_list: if decorator.func.attr == 'idempotent_id': tests['id-' + decorator.args[0].s] = \ module_name + "." + node.name + "." + \ classnode.name return tests def run(): parser = argparse.ArgumentParser() parser.add_argument('--guideline', action='store', dest='guideline_file', default='next.json', type=str, help='The name of the guideline file to to check') parser.add_argument('--testlib', action='store', dest='testlib', default='tempest', type=str, help='The test library, in the PYTHONPATH, to ' 'check the consistency of the guideline ' 'against') args = parser.parse_args() guideline = load_guideline(args.guideline_file) required = get_required_tests(guideline) tests = get_tests(args.testlib) missing_uuids = [] missing_tests = {} for test in required: uuid = test[0] testnames = test[1] if uuid not in tests: missing_uuids.append(test) else: if tests[uuid] not in testnames: missing_tests[uuid] = test exit_code = 0 if len(missing_uuids) > 0: exit_code = 1 print("### Idempotent ID Errors Detected. To resolve these errors, " "fix the uuid name (format id-) in the guideline to " "match the id in the test suite:") for test in missing_uuids: print("Idempotent ID in guideline '%s' does not appear in test " "library '%s'\n" " idempotent_id:\n" " %s\n" " names: " % (args.guideline_file, args.testlib, test[0])) for testname in test[1]: print(" %s" % (testname)) print("") if len(missing_tests) > 0: exit_code = 1 print("### Test Name Errors Detected. " "To resolve these errors, update " "the Interop guideline with the missing " "test names:") for uuid in missing_tests: print("Test found in test library '%s'\n" " idempotent_id:\n" " %s\n" " name:\n" " %s\n" "Entry in guideline '%s'\n" " idempotent_id:\n" " %s\n" " names: " % (args.testlib, uuid, tests[uuid], args.guideline_file, missing_tests[uuid][0])) for testname in missing_tests[uuid][1]: print(" %s" % (testname)) print("") sys.exit(exit_code) if __name__ == '__main__': run()