diff --git a/.zuul.d/zuul.yaml b/.zuul.d/zuul.yaml index c3858b9e..f3fd96e1 100644 --- a/.zuul.d/zuul.yaml +++ b/.zuul.d/zuul.yaml @@ -79,8 +79,7 @@ # watcher-api did not start last time voting: false - rally-task-zaqar - - rally-verify-tempest: - voting: false + - rally-verify-tempest gate: jobs: - rally-tox-cover @@ -117,6 +116,7 @@ - rally_openstack/task/scenarios/neutron/network.py - tests/ci/playbooks - rally-task-zaqar + - rally-verify-tempest post: jobs: - rally-openstack-docker-build-and-push: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 33ef4f56..1a290c5d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -16,6 +16,15 @@ Changelog .. Release notes for existing releases are MUTABLE! If there is something that was missed or can be improved, feel free to change it! + +unreleased +---------- + +Fixed +~~~~~ + +Compatibility with Tempest >= 27 + [2.1.0] - 2020-11-03 -------------------- diff --git a/rally_openstack/verification/tempest/config.ini b/rally_openstack/verification/tempest/config.ini index 4d64e8f6..26458d2b 100644 --- a/rally_openstack/verification/tempest/config.ini +++ b/rally_openstack/verification/tempest/config.ini @@ -46,7 +46,6 @@ lock_path = instance_type = [scenario] -img_dir = img_file = [service_available] diff --git a/rally_openstack/verification/tempest/context.py b/rally_openstack/verification/tempest/context.py index acc7e2d0..4ee07778 100644 --- a/rally_openstack/verification/tempest/context.py +++ b/rally_openstack/verification/tempest/context.py @@ -64,6 +64,20 @@ class TempestContext(context.VerifierContext): self._created_flavors = [] self._created_networks = [] + def _configure_img_options(self): + try: + tempest_major_version = int(self.verifier.version.split(".", 1)[0]) + except ValueError: + # use latest flow by default + tempest_major_version = 27 + if tempest_major_version < 27: + self._configure_option("scenario", "img_dir", self.data_dir) + img_file = self.image_name + else: + img_file = self.data_dir + "/" + self.image_name + self._configure_option("scenario", "img_file", img_file, + helper_method=self._download_image) + def setup(self): self.conf.read(self.conf_path) @@ -75,9 +89,7 @@ class TempestContext(context.VerifierContext): os.path.join(self.data_dir, "tempest.log")) self._configure_option("oslo_concurrency", "lock_path", os.path.join(self.data_dir, "lock_files")) - self._configure_option("scenario", "img_dir", self.data_dir) - self._configure_option("scenario", "img_file", self.image_name, - helper_method=self._download_image) + self._configure_img_options() self._configure_option("compute", "image_ref", helper_method=self._discover_or_create_image) self._configure_option("compute", "image_ref_alt", diff --git a/tests/ci/rally_verify.py b/tests/ci/rally_verify.py index 949a1f77..7559923f 100644 --- a/tests/ci/rally_verify.py +++ b/tests/ci/rally_verify.py @@ -80,10 +80,10 @@ class Step(object): "skipped or finished with an error." % (self.name, results[self.DEPENDS_ON].name)) stdout_file = self._generate_path( - "%s.txt.gz" % self.__class__.__name__) + "%s.txt" % self.__class__.__name__) self.result["output_file"] = self._write_file( - stdout_file, msg, compress=True) + stdout_file, msg, compress=False) return False return True @@ -99,9 +99,9 @@ class Step(object): self.result["cmd"] = cmd self.result["status"], self.result["output"] = self.call_rally(cmd) - stdout_file = self._generate_path("%s.txt.gz" % cmd) + stdout_file = self._generate_path("%s.txt" % cmd) self.result["output_file"] = self._write_file( - stdout_file, self.result["output"], compress=True) + stdout_file, self.result["output"], compress=False) @classmethod def _generate_path(cls, root): @@ -317,12 +317,34 @@ class RunVerification(Step): SKIP_TESTS = { "tempest.api.compute.flavors.test_flavors.FlavorsV2TestJSON." "test_get_flavor[id-1f12046b-753d-40d2-abb6-d8eb8b30cb2f,smoke]": - "This test was skipped intentionally"} + "This test was skipped intentionally", + } XFAIL_TESTS = { - "tempest.api.compute.servers.test_server_actions." - "ServerActionsTestJSON.test_get_vnc_console" - "[id-c6bc11bf-592e-4015-9319-1c98dc64daf5]": - "This test fails because 'novnc' console type is unavailable"} + "tempest.scenario.test_dashboard_basic_ops" + ".TestDashboardBasicOps.test_basic_scenario" + "[dashboard,id-4f8851b1-0e69-482b-b63b-84c6e76f6c80,smoke]": + "Fails for unknown reason", + + "tempest.api.compute.servers.test_attach_interfaces" + ".AttachInterfacesUnderV243Test.test_add_remove_fixed_ip" + "[id-c7e0e60b-ee45-43d0-abeb-8596fd42a2f9,network,smoke]": + "Fails for unknown reason", + + "tempest.scenario.test_network_basic_ops" + ".TestNetworkBasicOps.test_network_basic_ops" + "[compute,id-f323b3ba-82f8-4db7-8ea6-6a895869ec49,network,smoke]": + "Fails for unknown reason", + + "tempest.scenario.test_server_basic_ops" + ".TestServerBasicOps.test_server_basic_ops" + "[compute,id-7fff3fb3-91d8-4fd0-bd7d-0204f1f180ba,network,smoke]": + "Fails for unknown reason", + + "tempest.api.compute.servers.test_server_actions" + ".ServerActionsTestJSON.test_reboot_server_hard" + "[id-2cb1baf6-ac8d-4429-bf0d-ba8a0ba53e32,smoke]": + "Fails for unknown reason" + } def setUp(self): self.CALL_ARGS["tag"] = "tag-1 tag-2" @@ -396,14 +418,6 @@ class ReportVerificationMixin(Step): print(self.COMMAND % self.CALL_ARGS) self.result["out"] = "" - def run(self): - super(ReportVerificationMixin, self).run() - creport = "%s.gz" % self.CALL_ARGS["out"] - with open(self.CALL_ARGS["out"], "rb") as f_in: - with gzip.open(creport, "wb") as f_out: - f_out.writelines(f_in) - self.result["out"] = creport - class HtmlVerificationReport(ReportVerificationMixin): """Generate HTML report for verification(s).""" diff --git a/tests/unit/verification/tempest/test_context.py b/tests/unit/verification/tempest/test_context.py index 88c9fa47..9e503b9e 100644 --- a/tests/unit/verification/tempest/test_context.py +++ b/tests/unit/verification/tempest/test_context.py @@ -369,7 +369,7 @@ class TempestContextTestCase(test.TestCase): def test_setup(self, mock_create_dir, mock__create_tempest_roles, mock__configure_option, mock_open): - verifier = mock.Mock(env=self.env) + verifier = mock.Mock(env=self.env, version="27.1.0") verifier.manager.home_dir = "/p/a/t/h" # case #1: no neutron and heat @@ -387,8 +387,7 @@ class TempestContextTestCase(test.TestCase): self.assertEqual( [mock.call("DEFAULT", "log_file", "/p/a/t/h/tempest.log"), mock.call("oslo_concurrency", "lock_path", "/p/a/t/h/lock_files"), - mock.call("scenario", "img_dir", "/p/a/t/h"), - mock.call("scenario", "img_file", ctx.image_name, + mock.call("scenario", "img_file", "/p/a/t/h/" + ctx.image_name, helper_method=ctx._download_image), mock.call("compute", "image_ref", helper_method=ctx._discover_or_create_image), @@ -427,8 +426,7 @@ class TempestContextTestCase(test.TestCase): self.assertEqual([ mock.call("DEFAULT", "log_file", "/p/a/t/h/tempest.log"), mock.call("oslo_concurrency", "lock_path", "/p/a/t/h/lock_files"), - mock.call("scenario", "img_dir", "/p/a/t/h"), - mock.call("scenario", "img_file", ctx.image_name, + mock.call("scenario", "img_file", "/p/a/t/h/" + ctx.image_name, helper_method=ctx._download_image), mock.call("compute", "image_ref", helper_method=ctx._discover_or_create_image), @@ -449,3 +447,16 @@ class TempestContextTestCase(test.TestCase): flv_ram=config.CONF.openstack.heat_instance_type_ram, flv_disk=config.CONF.openstack.heat_instance_type_disk) ], mock__configure_option.call_args_list) + + # case 3: tempest is old. + verifier.version = "17.0.0" + ctx = context.TempestContext({"verifier": verifier}) + ctx.conf = mock.Mock() + ctx.setup() + mock__configure_option.assert_has_calls( + [ + mock.call("scenario", "img_dir", "/p/a/t/h"), + mock.call("scenario", "img_file", ctx.image_name, + helper_method=ctx._download_image) + ] + )