96 lines
2.9 KiB
Python
Executable File
96 lines
2.9 KiB
Python
Executable File
#!/usr/bin/env python
|
|
|
|
import functools
|
|
import re
|
|
import sys
|
|
|
|
import subunit
|
|
import testtools
|
|
|
|
SUCCESS = []
|
|
SKIPS = []
|
|
FAILS = []
|
|
ADDPROP_FAIL = []
|
|
|
|
|
|
def find_additionalProperties_in_traceback(traceback):
|
|
error_msg_re = re.compile(
|
|
"^tempest.lib.exceptions.InvalidHTTPResponseBody\:")
|
|
found_error_msg = False
|
|
error_msg = []
|
|
for line in traceback:
|
|
temp_line = line.strip()
|
|
if not temp_line:
|
|
continue
|
|
if found_error_msg:
|
|
error_msg.append(line)
|
|
if error_msg_re.search(temp_line):
|
|
found_error_msg = True
|
|
continue
|
|
|
|
if not found_error_msg and not error_msg:
|
|
return False
|
|
else:
|
|
properties_regex = re.compile(
|
|
"^Failed validating 'additionalProperties' in schema")
|
|
# TODO(mtreinish): Add more specific checks to limit the allowed
|
|
# APIs with additional properties
|
|
if not properties_regex.search(error_msg[1].strip()):
|
|
return False
|
|
else:
|
|
return error_msg
|
|
|
|
|
|
def show_outcome(stream, test):
|
|
global RESULTS
|
|
status = test['status']
|
|
if status == 'exists':
|
|
return
|
|
if status == 'fail':
|
|
for raw_name in test['details']:
|
|
name = raw_name.split(':')[0]
|
|
detail = test['details'][raw_name]
|
|
if detail.content_type.type == 'test':
|
|
detail.content_type.type = 'text'
|
|
if name == 'traceback':
|
|
traceback = detail.as_text().split('\n')
|
|
res = find_additionalProperties_in_traceback(traceback)
|
|
if isinstance(res, list):
|
|
title = (
|
|
"%s Failed with AdditionalProperties jsonschema "
|
|
"failure" % test['id'])
|
|
stream.write("\n%s\n%s\n" % (title, ('~' * len(title))))
|
|
for line in res:
|
|
line = line.encode('utf8')
|
|
stream.write("%s\n" % line)
|
|
stream.write('\n\n')
|
|
ADDPROP_FAIL.append(test)
|
|
break
|
|
else:
|
|
FAILS.append(test)
|
|
elif status == 'success' or status == 'xfail':
|
|
SUCCESS.append(test)
|
|
elif status == 'skip':
|
|
SKIPS.append(test)
|
|
|
|
|
|
stream = subunit.ByteStreamToStreamResult(
|
|
sys.stdin, non_subunit_name='stdout')
|
|
outcome = testtools.StreamToDict(
|
|
functools.partial(show_outcome,
|
|
sys.stdout))
|
|
summary = testtools.StreamSummary()
|
|
result = testtools.CopyStreamResult([outcome, summary])
|
|
result.startTestRun()
|
|
try:
|
|
stream.run(result)
|
|
finally:
|
|
result.stopTestRun()
|
|
|
|
print("\n\n------------------------------------------------------------------")
|
|
print("%s Tests Failed" % len(FAILS))
|
|
print("%s Tests Failed with AdditionalProperties" % len(ADDPROP_FAIL))
|
|
print("%s Tests Skipped" % len(SKIPS))
|
|
print("%s Tests Passed" % len(SUCCESS))
|
|
print("To see the full details run this subunit stream through subunit-trace")
|