From c38e18b343505f16a74a97b748362fa7f1a01e57 Mon Sep 17 00:00:00 2001 From: Chris Hoge Date: Wed, 22 Jun 2016 16:36:25 -0700 Subject: [PATCH] Testing waiver for vendors using Nova 2.0 API with additional properties This waiver allows vendors who are using the Nova 2.0 API with additional properties to disable strict response checking when testing products for the OpenStack Powered program in 2016. Co-Authored-By: Matthew Treinish Change-Id: Ibd4be9cfb86d26df7f484a1e9c2d2e46275203cc --- .../additional_properties_waiver.rst | 88 +++++++++++++++++ .../find_additional_properties.py | 94 +++++++++++++++++++ 2 files changed, 182 insertions(+) create mode 100644 working_materials/additional_properties_waiver.rst create mode 100644 working_materials/find_additional_properties.py diff --git a/working_materials/additional_properties_waiver.rst b/working_materials/additional_properties_waiver.rst new file mode 100644 index 00000000..09356cd4 --- /dev/null +++ b/working_materials/additional_properties_waiver.rst @@ -0,0 +1,88 @@ +============================ +Additional Properties Waiver +============================ + +In mid-2015, the OpenStack QA team implemented strict response +checking as an implementation detail and enforcement of Nova +microversions. Microversions, in development since the Kilo release of +OpenStack, were designed to allow for backwards compatible additions +to the API, giving both client and server the option to request and +receive responses within a known range of capabilities. In support +of the interoperability goals of microversions and compatbility between +OpenStack clouds, the QA team introduced strict API response checking of +Nova calls as part of tempest-lib. + +Prior to this change, clouds running the Nova 2.0 API could take +advantage of a mechanism to add extensions to the Nova endpoint, and +some also sent additional data back in their Nova responses. These clouds +passed interoperability testing when the DefCore interoperability testing +program was launched for the OpenStack Powered program. After strict +response checking was added to Tempest, these clouds failed interop +testing. + +To address this issue, and the challenges vendors have in updating their +products to match upstream API changes, this proposal offers a means for +vendors to pass the DefCore interoperability tests while proving +compatibility for required capabilities. + +There is a natural tension between the forward motion of upstream +development and the product requirements of downstream deployments. This +proposal attempts to reconcile that tension by extending the time that +vendors will be required to remove additional properties and replace +those features with alternatives. Possible resolutions for downstream +products include, but are not limited to: + +#. Removal of all additional properties. +#. Contributing micro-version changes upstream to capture additional + properties. +#. Using custom HTTP headers to request additional properties, to be + used by custom clients or tools. +#. Deploying additional endpoints that return unmodified responses. +#. Remaining on the Nova v/2.0 API, which has been removed from the + Newton release of OpenStack + +This waiver program will cover the 2015.07, 2016.01, and 2016.08 DefCore +guidelines, and give downstream vendors a year to work internally +and within the ecosystem to update their products before re-verifying +their products. + +It's important to note that the Nova team has for two years been +broadcasting their intentions[1][2][3], offering microversions as an +interoperable way to add new data, and has removed the 2.0 API and +extensions code from the Newton release. Although no known clients +implement strict response checking (except for the Tempest client), +it is clearly the direction that upsteam OpenStack developers have +signaled. + +================= +Details of Waiver +================= + +#. Products appyling for the OpenStack Powered Trademark in 2016 may + request the waiver by submitting subunit data from their Tempest run + that can be analyzed by the `find_additional_properties.py` script + from the DefCore repository. This script will identify tests that + failed because of additional properties. The vendor will then need + to modify tempest-lib[4] to remove additional checks on the impacted + APIs. Development is beginning within the refstack-client project[5] + to automate generation of a patch for tempest-lib. + +#. Products that use additional properties in Nova API responses will be + clearly identified in the OpenStack Marketplace, with the product + listing showing which APIs have included additional response data. + Products using additional data will be restricted to the Nova 2.0 API. + +#. Beginning with the 2017.01 release of the DefCore guidelines, this + waiver program will no longer be in force, unless 'additional + properties' is listed as an acceptable implementation using the Nova + 2.0 API in the forthcoming DefCore capabilities. All other new + products must pass upstream testing. + +#. Aside from additional properties, no products may change the json API + response in any other way. + +[1] http://lists.openstack.org/pipermail/openstack-dev/2015-February/057613.html +[2] https://specs.openstack.org/openstack/nova-specs/specs/kilo/implemented/api-microversions.html +[3] http://lists.openstack.org/pipermail/openstack-dev/2015-March/059576.html +[4] https://github.com/openstack/tempest/tree/master/tempest/lib/api_schema/response/compute +[5] http://git.openstack.org/cgit/openstack/refstack-client/ diff --git a/working_materials/find_additional_properties.py b/working_materials/find_additional_properties.py new file mode 100644 index 00000000..954d0622 --- /dev/null +++ b/working_materials/find_additional_properties.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python + +import functools +import re +import sys + +import subunit +import testtools + +SUCCESS = [] +SKIPS = [] +FAILS = [] +ADDPROP_FAIL = [] + + +def find_additionalProperties_in_traceback(traceback): + error_msg_re = re.compile( + "^tempest.lib.exceptions.InvalidHTTPResponseBody\:") + found_error_msg = False + error_msg = [] + for line in traceback: + temp_line = line.strip() + if not temp_line: + continue + if found_error_msg: + error_msg.append(line) + if error_msg_re.search(temp_line): + found_error_msg = True + continue + + if not found_error_msg and not error_msg: + return False + else: + properties_regex = re.compile( + "^Failed validating 'additionalProperties' in schema") + # TODO(mtreinish): Add more specific checks to limit the allowed + # APIs with additional properties + if not properties_regex.search(error_msg[1].strip()): + return False + else: + return error_msg + + +def show_outcome(stream, test): + global RESULTS + status = test['status'] + if status == 'exists': + returnmime + if status == 'fail': + for raw_name in test['details']: + name = raw_name.split(':')[0] + detail = test['details'][raw_name] + if detail.content_type.type == 'test': + detail.content_type.type = 'text' + if name == 'traceback': + traceback = detail.as_text().split('\n') + res = find_additionalProperties_in_traceback(traceback) + if isinstance(res, list): + title = ( + "%s Failed with AdditionalProperties jsonschema " + "failure" % test['id']) + stream.write("\n%s\n%s\n" % (title, ('~' * len(title)))) + for line in res: + line = line.encode('utf8') + stream.write("%s\n" % line) + stream.write('\n\n') + ADDPROP_FAIL.append(test) + break + else: + FAILS.append(test) + elif status == 'success' or status == 'xfail': + SUCCESS.append(test) + elif status == 'skip': + SKIPS.append(test) + +stream = subunit.ByteStreamToStreamResult( + sys.stdin, non_subunit_name='stdout') +outcome = testtools.StreamToDict( + functools.partial(show_outcome, + sys.stdout)) +summary = testtools.StreamSummary() +result = testtools.CopyStreamResult([outcome, summary]) +result.startTestRun() +try: + stream.run(result) +finally: + result.stopTestRun() + +print("\n\n------------------------------------------------------------------") +print("%s Tests Failed" % len(FAILS)) +print("%s Tests Failed with AdditionalProperties" % len(ADDPROP_FAIL)) +print("%s Tests Skipped" % len(SKIPS)) +print("%s Tests Passed" % len(SUCCESS)) +print("To see the full details run this subunit stream through subunit-trace")