diff --git a/working_materials/additional_properties_waiver.rst b/working_materials/additional_properties_waiver.rst new file mode 100644 index 00000000..09356cd4 --- /dev/null +++ b/working_materials/additional_properties_waiver.rst @@ -0,0 +1,88 @@ +============================ +Additional Properties Waiver +============================ + +In mid-2015, the OpenStack QA team implemented strict response +checking as an implementation detail and enforcement of Nova +microversions. Microversions, in development since the Kilo release of +OpenStack, were designed to allow for backwards compatible additions +to the API, giving both client and server the option to request and +receive responses within a known range of capabilities. In support +of the interoperability goals of microversions and compatbility between +OpenStack clouds, the QA team introduced strict API response checking of +Nova calls as part of tempest-lib. + +Prior to this change, clouds running the Nova 2.0 API could take +advantage of a mechanism to add extensions to the Nova endpoint, and +some also sent additional data back in their Nova responses. These clouds +passed interoperability testing when the DefCore interoperability testing +program was launched for the OpenStack Powered program. After strict +response checking was added to Tempest, these clouds failed interop +testing. + +To address this issue, and the challenges vendors have in updating their +products to match upstream API changes, this proposal offers a means for +vendors to pass the DefCore interoperability tests while proving +compatibility for required capabilities. + +There is a natural tension between the forward motion of upstream +development and the product requirements of downstream deployments. This +proposal attempts to reconcile that tension by extending the time that +vendors will be required to remove additional properties and replace +those features with alternatives. Possible resolutions for downstream +products include, but are not limited to: + +#. Removal of all additional properties. +#. Contributing micro-version changes upstream to capture additional + properties. +#. Using custom HTTP headers to request additional properties, to be + used by custom clients or tools. +#. Deploying additional endpoints that return unmodified responses. +#. Remaining on the Nova v/2.0 API, which has been removed from the + Newton release of OpenStack + +This waiver program will cover the 2015.07, 2016.01, and 2016.08 DefCore +guidelines, and give downstream vendors a year to work internally +and within the ecosystem to update their products before re-verifying +their products. + +It's important to note that the Nova team has for two years been +broadcasting their intentions[1][2][3], offering microversions as an +interoperable way to add new data, and has removed the 2.0 API and +extensions code from the Newton release. Although no known clients +implement strict response checking (except for the Tempest client), +it is clearly the direction that upsteam OpenStack developers have +signaled. + +================= +Details of Waiver +================= + +#. Products appyling for the OpenStack Powered Trademark in 2016 may + request the waiver by submitting subunit data from their Tempest run + that can be analyzed by the `find_additional_properties.py` script + from the DefCore repository. This script will identify tests that + failed because of additional properties. The vendor will then need + to modify tempest-lib[4] to remove additional checks on the impacted + APIs. Development is beginning within the refstack-client project[5] + to automate generation of a patch for tempest-lib. + +#. Products that use additional properties in Nova API responses will be + clearly identified in the OpenStack Marketplace, with the product + listing showing which APIs have included additional response data. + Products using additional data will be restricted to the Nova 2.0 API. + +#. Beginning with the 2017.01 release of the DefCore guidelines, this + waiver program will no longer be in force, unless 'additional + properties' is listed as an acceptable implementation using the Nova + 2.0 API in the forthcoming DefCore capabilities. All other new + products must pass upstream testing. + +#. Aside from additional properties, no products may change the json API + response in any other way. + +[1] http://lists.openstack.org/pipermail/openstack-dev/2015-February/057613.html +[2] https://specs.openstack.org/openstack/nova-specs/specs/kilo/implemented/api-microversions.html +[3] http://lists.openstack.org/pipermail/openstack-dev/2015-March/059576.html +[4] https://github.com/openstack/tempest/tree/master/tempest/lib/api_schema/response/compute +[5] http://git.openstack.org/cgit/openstack/refstack-client/ diff --git a/working_materials/find_additional_properties.py b/working_materials/find_additional_properties.py new file mode 100644 index 00000000..954d0622 --- /dev/null +++ b/working_materials/find_additional_properties.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python + +import functools +import re +import sys + +import subunit +import testtools + +SUCCESS = [] +SKIPS = [] +FAILS = [] +ADDPROP_FAIL = [] + + +def find_additionalProperties_in_traceback(traceback): + error_msg_re = re.compile( + "^tempest.lib.exceptions.InvalidHTTPResponseBody\:") + found_error_msg = False + error_msg = [] + for line in traceback: + temp_line = line.strip() + if not temp_line: + continue + if found_error_msg: + error_msg.append(line) + if error_msg_re.search(temp_line): + found_error_msg = True + continue + + if not found_error_msg and not error_msg: + return False + else: + properties_regex = re.compile( + "^Failed validating 'additionalProperties' in schema") + # TODO(mtreinish): Add more specific checks to limit the allowed + # APIs with additional properties + if not properties_regex.search(error_msg[1].strip()): + return False + else: + return error_msg + + +def show_outcome(stream, test): + global RESULTS + status = test['status'] + if status == 'exists': + returnmime + if status == 'fail': + for raw_name in test['details']: + name = raw_name.split(':')[0] + detail = test['details'][raw_name] + if detail.content_type.type == 'test': + detail.content_type.type = 'text' + if name == 'traceback': + traceback = detail.as_text().split('\n') + res = find_additionalProperties_in_traceback(traceback) + if isinstance(res, list): + title = ( + "%s Failed with AdditionalProperties jsonschema " + "failure" % test['id']) + stream.write("\n%s\n%s\n" % (title, ('~' * len(title)))) + for line in res: + line = line.encode('utf8') + stream.write("%s\n" % line) + stream.write('\n\n') + ADDPROP_FAIL.append(test) + break + else: + FAILS.append(test) + elif status == 'success' or status == 'xfail': + SUCCESS.append(test) + elif status == 'skip': + SKIPS.append(test) + +stream = subunit.ByteStreamToStreamResult( + sys.stdin, non_subunit_name='stdout') +outcome = testtools.StreamToDict( + functools.partial(show_outcome, + sys.stdout)) +summary = testtools.StreamSummary() +result = testtools.CopyStreamResult([outcome, summary]) +result.startTestRun() +try: + stream.run(result) +finally: + result.stopTestRun() + +print("\n\n------------------------------------------------------------------") +print("%s Tests Failed" % len(FAILS)) +print("%s Tests Failed with AdditionalProperties" % len(ADDPROP_FAIL)) +print("%s Tests Skipped" % len(SKIPS)) +print("%s Tests Passed" % len(SUCCESS)) +print("To see the full details run this subunit stream through subunit-trace")