diff --git a/rally/plugins/openstack/context/swift/__init__.py b/rally/plugins/openstack/context/swift/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/rally/plugins/openstack/context/swift/objects.py b/rally/plugins/openstack/context/swift/objects.py new file mode 100644 index 00000000..c648871f --- /dev/null +++ b/rally/plugins/openstack/context/swift/objects.py @@ -0,0 +1,100 @@ +# Copyright 2015: Cisco Systems, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from rally.common.i18n import _ +from rally.common import log as logging +from rally.common import utils as rutils +from rally import consts +from rally import exceptions +from rally.plugins.openstack.context.swift import utils as swift_utils +from rally.task import context + +LOG = logging.getLogger(__name__) + + +@context.configure(name="swift_objects", order=360) +class SwiftObjectGenerator(swift_utils.SwiftObjectMixin, context.Context): + CONFIG_SCHEMA = { + "type": "object", + "$schema": consts.JSON_SCHEMA, + "properties": { + "containers_per_tenant": { + "type": "integer", + "minimum": 1 + }, + "objects_per_container": { + "type": "integer", + "minimum": 1 + }, + "object_size": { + "type": "integer", + "minimum": 1 + }, + "resource_management_workers": { + "type": "integer", + "minimum": 1 + } + }, + "additionalProperties": False + } + + DEFAULT_CONFIG = { + "containers_per_tenant": 1, + "objects_per_container": 1, + "object_size": 1024, + "resource_management_workers": 30 + } + + @rutils.log_task_wrapper(LOG.info, _("Enter context: `swift_objects`")) + def setup(self): + """Create containers and objects, using the broker pattern.""" + threads = self.config["resource_management_workers"] + + containers_per_tenant = self.config["containers_per_tenant"] + containers_num = len(self.context["tenants"]) * containers_per_tenant + LOG.debug("Creating %d containers using %d threads." % (containers_num, + threads)) + containers_count = len(self._create_containers(self.context, + containers_per_tenant, + threads)) + if containers_count != containers_num: + raise exceptions.ContextSetupFailure( + ctx_name=self.get_name(), + msg=_("Failed to create the requested number of containers, " + "expected %(expected)s but got %(actual)s.") + % {"expected": containers_num, "actual": containers_count}) + + objects_per_container = self.config["objects_per_container"] + objects_num = containers_num * objects_per_container + LOG.debug("Creating %d objects using %d threads." % (objects_num, + threads)) + objects_count = len(self._create_objects(self.context, + objects_per_container, + self.config["object_size"], + threads)) + if objects_count != objects_num: + raise exceptions.ContextSetupFailure( + ctx_name=self.get_name(), + msg=_("Failed to create the requested number of objects, " + "expected %(expected)s but got %(actual)s.") + % {"expected": objects_num, "actual": objects_count}) + + @rutils.log_task_wrapper(LOG.info, _("Exit context: `swift_objects`")) + def cleanup(self): + """Delete containers and objects, using the broker pattern.""" + threads = self.config["resource_management_workers"] + + self._delete_objects(self.context, threads) + self._delete_containers(self.context, threads) diff --git a/rally/plugins/openstack/context/swift/utils.py b/rally/plugins/openstack/context/swift/utils.py new file mode 100644 index 00000000..b3c1b6ed --- /dev/null +++ b/rally/plugins/openstack/context/swift/utils.py @@ -0,0 +1,148 @@ +# Copyright 2015: Cisco Systems, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import tempfile + +from rally.common import broker +from rally.common import utils as rutils +from rally.plugins.openstack.scenarios.swift import utils as swift_utils + + +class SwiftObjectMixin(object): + """Mix-in method for Swift Object Context.""" + + def _create_containers(self, context, containers_per_tenant, threads): + """Create containers and store results in Rally context. + + :param context: dict, Rally context environment + :param containers_per_tenant: int, number of containers to create + per tenant + :param threads: int, number of threads to use for broker pattern + + :returns: list of tuples containing (account, container) + """ + containers = [] + + def publish(queue): + for user, tenant_id in (rutils.iterate_per_tenants( + context.get("users", []))): + context["tenants"][tenant_id]["containers"] = [] + for i in range(containers_per_tenant): + args = (user, context["tenants"][tenant_id]["containers"]) + queue.append(args) + + def consume(cache, args): + user, tenant_containers = args + if user["id"] not in cache: + cache[user["id"]] = swift_utils.SwiftScenario({"user": user}) + container_name = cache[user["id"]]._create_container() + tenant_containers.append({"user": user, + "container": container_name, + "objects": []}) + containers.append((user["tenant_id"], container_name)) + + broker.run(publish, consume, threads) + + return containers + + def _create_objects(self, context, objects_per_container, object_size, + threads): + """Create objects and store results in Rally context. + + :param context: dict, Rally context environment + :param objects_per_container: int, number of objects to create + per container + :param object_size: int, size of created swift objects in byte + :param threads: int, number of threads to use for broker pattern + + :returns: list of tuples containing (account, container, object) + """ + objects = [] + + with tempfile.TemporaryFile() as dummy_file: + # set dummy file to specified object size + dummy_file.truncate(object_size) + + def publish(queue): + for tenant_id in context["tenants"]: + containers = context["tenants"][tenant_id]["containers"] + for container in containers: + for i in range(objects_per_container): + queue.append(container) + + def consume(cache, container): + user = container["user"] + if user["id"] not in cache: + cache[user["id"]] = swift_utils.SwiftScenario( + {"user": user}) + dummy_file.seek(0) + object_name = cache[user["id"]]._upload_object( + container["container"], + dummy_file)[1] + container["objects"].append(object_name) + objects.append((user["tenant_id"], container["container"], + object_name)) + + broker.run(publish, consume, threads) + + return objects + + def _delete_containers(self, context, threads): + """Delete containers created by Swift context and update Rally context. + + :param context: dict, Rally context environment + :param threads: int, number of threads to use for broker pattern + """ + def publish(queue): + for tenant_id in context["tenants"]: + containers = context["tenants"][tenant_id]["containers"] + for container in containers[:]: + args = container, containers + queue.append(args) + + def consume(cache, args): + container, tenant_containers = args + user = container["user"] + if user["id"] not in cache: + cache[user["id"]] = swift_utils.SwiftScenario({"user": user}) + cache[user["id"]]._delete_container(container["container"]) + tenant_containers.remove(container) + + broker.run(publish, consume, threads) + + def _delete_objects(self, context, threads): + """Delete objects created by Swift context and update Rally context. + + :param context: dict, Rally context environment + :param threads: int, number of threads to use for broker pattern + """ + def publish(queue): + for tenant_id in context["tenants"]: + containers = context["tenants"][tenant_id]["containers"] + for container in containers: + for object_name in container["objects"][:]: + args = object_name, container + queue.append(args) + + def consume(cache, args): + object_name, container = args + user = container["user"] + if user["id"] not in cache: + cache[user["id"]] = swift_utils.SwiftScenario({"user": user}) + cache[user["id"]]._delete_object(container["container"], + object_name) + container["objects"].remove(object_name) + + broker.run(publish, consume, threads) diff --git a/tests/unit/plugins/openstack/context/swift/__init__.py b/tests/unit/plugins/openstack/context/swift/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/plugins/openstack/context/swift/test_objects.py b/tests/unit/plugins/openstack/context/swift/test_objects.py new file mode 100644 index 00000000..f339022c --- /dev/null +++ b/tests/unit/plugins/openstack/context/swift/test_objects.py @@ -0,0 +1,204 @@ +# Copyright 2015: Cisco Systems, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from rally import exceptions +from rally.plugins.openstack.context.swift import objects +from tests.unit import test + + +class SwiftObjectGeneratorTestCase(test.TestCase): + + @mock.patch("rally.osclients.Clients") + def test_setup(self, mock_clients): + containers_per_tenant = 2 + objects_per_container = 7 + context = { + "config": { + "swift_objects": { + "containers_per_tenant": containers_per_tenant, + "objects_per_container": objects_per_container, + "object_size": 1024, + "resource_management_workers": 10 + } + }, + "task": {"uuid": "id123"}, + "tenants": { + "t1": {"name": "t1_name"}, + "t2": {"name": "t2_name"} + }, + "users": [ + {"id": "u1", "tenant_id": "t1", "endpoint": "e1"}, + {"id": "u2", "tenant_id": "t2", "endpoint": "e2"} + ] + } + + objects_ctx = objects.SwiftObjectGenerator(context) + objects_ctx.setup() + + for tenant_id in context["tenants"]: + containers = context["tenants"][tenant_id]["containers"] + self.assertEqual(containers_per_tenant, len(containers)) + for container in containers: + self.assertIn("rally_container_", container["container"]) + self.assertEqual(objects_per_container, + len(container["objects"])) + for obj in container["objects"]: + self.assertIn("rally_object_", obj) + + @mock.patch("rally.osclients.Clients") + @mock.patch("rally.plugins.openstack.context.swift.utils." + "swift_utils.SwiftScenario") + def test_cleanup(self, mock_swift_scenario, mock_clients): + context = { + "config": { + "swift_objects": { + "resource_management_workers": 1 + } + }, + "task": {"uuid": "id123"}, + "tenants": { + "t1": { + "name": "t1_name", + "containers": [ + {"user": {"id": "u1", "tenant_id": "t1", + "endpoint": "e1"}, + "container": "c1", + "objects": ["o1", "o2", "o3"]} + ] + }, + "t2": { + "name": "t2_name", + "containers": [ + {"user": {"id": "u2", "tenant_id": "t2", + "endpoint": "e2"}, + "container": "c2", + "objects": ["o4", "o5", "o6"]} + ] + } + } + } + + objects_ctx = objects.SwiftObjectGenerator(context) + objects_ctx.cleanup() + + expected_containers = ["c1", "c2"] + mock_swift_scenario.return_value._delete_container.assert_has_calls( + [mock.call(con) for con in expected_containers], any_order=True) + + expected_objects = [("c1", "o1"), ("c1", "o2"), ("c1", "o3"), + ("c2", "o4"), ("c2", "o5"), ("c2", "o6")] + mock_swift_scenario.return_value._delete_object.assert_has_calls( + [mock.call(con, obj) for con, obj in expected_objects], + any_order=True) + + for tenant_id in context["tenants"]: + self.assertEqual(0, + len(context["tenants"][tenant_id]["containers"])) + + @mock.patch("rally.osclients.Clients") + def test_setup_failure_clients_put_container(self, mock_clients): + context = { + "config": { + "swift_objects": { + "containers_per_tenant": 2, + "object_size": 10, + "resource_management_workers": 5 + } + }, + "task": {"uuid": "id123"}, + "tenants": { + "t1": {"name": "t1_name"}, + "t2": {"name": "t2_name"} + }, + "users": [ + {"id": "u1", "tenant_id": "t1", "endpoint": "e1"}, + {"id": "u2", "tenant_id": "t2", "endpoint": "e2"} + ] + } + mock_swift = mock_clients.return_value.swift.return_value + mock_swift.put_container.side_effect = [Exception, True, + Exception, Exception] + objects_ctx = objects.SwiftObjectGenerator(context) + self.assertRaisesRegexp(exceptions.ContextSetupFailure, + "containers, expected 4 but got 1", + objects_ctx.setup) + + @mock.patch("rally.osclients.Clients") + def test_setup_failure_clients_put_object(self, mock_clients): + context = { + "task": {"uuid": "id123"}, + "tenants": { + "t1": {"name": "t1_name"}, + "t2": {"name": "t2_name"} + }, + "users": [ + {"id": "u1", "tenant_id": "t1", "endpoint": "e1"}, + {"id": "u2", "tenant_id": "t2", "endpoint": "e2"} + ] + } + mock_swift = mock_clients.return_value.swift.return_value + mock_swift.put_object.side_effect = [Exception, True] + objects_ctx = objects.SwiftObjectGenerator(context) + self.assertRaisesRegexp(exceptions.ContextSetupFailure, + "objects, expected 2 but got 1", + objects_ctx.setup) + + @mock.patch("rally.osclients.Clients") + def test_cleanup_failure_clients_delete_container(self, mock_clients): + context = { + "task": {"uuid": "id123"}, + "tenants": { + "t1": { + "name": "t1_name", + "containers": [ + {"user": {"id": "u1", "tenant_id": "t1", + "endpoint": "e1"}, + "container": "coooon", + "objects": []}] * 3 + } + } + } + mock_swift = mock_clients.return_value.swift.return_value + mock_swift.delete_container.side_effect = [True, True, Exception] + objects_ctx = objects.SwiftObjectGenerator(context) + objects_ctx.cleanup() + self.assertEqual(1, len(context["tenants"]["t1"]["containers"])) + + @mock.patch("rally.osclients.Clients") + def test_cleanup_failure_clients_delete_object(self, mock_clients): + context = { + "task": {"uuid": "id123"}, + "tenants": { + "t1": { + "name": "t1_name", + "containers": [ + {"user": {"id": "u1", "tenant_id": "t1", + "endpoint": "e1"}, + "container": "c1", + "objects": ["oooo"] * 3} + ] + } + } + } + mock_swift = mock_clients.return_value.swift.return_value + mock_swift.delete_object.side_effect = [True, Exception, True] + objects_ctx = objects.SwiftObjectGenerator(context) + objects_ctx._delete_containers = mock.MagicMock() + objects_ctx.cleanup() + self.assertEqual( + 1, sum([len(container["objects"]) + for container in context["tenants"]["t1"]["containers"]])) diff --git a/tests/unit/plugins/openstack/context/swift/test_utils.py b/tests/unit/plugins/openstack/context/swift/test_utils.py new file mode 100644 index 00000000..047b8b99 --- /dev/null +++ b/tests/unit/plugins/openstack/context/swift/test_utils.py @@ -0,0 +1,191 @@ +# Copyright 2015: Cisco Systems, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from rally.plugins.openstack.context.swift import utils +from tests.unit import test + + +class SwiftObjectMixinTestCase(test.TestCase): + + @mock.patch("rally.osclients.Clients") + def test__create_containers(self, mock_clients): + tenants = 2 + containers_per_tenant = 2 + context = { + "task": {"uuid": "id123"}, + "tenants": { + "1001": {"name": "t1_name"}, + "1002": {"name": "t2_name"} + }, + "users": [ + {"id": "u1", "tenant_id": "1001", "endpoint": "e1"}, + {"id": "u2", "tenant_id": "1002", "endpoint": "e2"} + ] + } + + mixin = utils.SwiftObjectMixin() + containers = mixin._create_containers(context, containers_per_tenant, + 15) + + self.assertEqual(tenants * containers_per_tenant, len(containers)) + for index, container in enumerate(sorted(containers)): + offset = int(index / containers_per_tenant) + 1 + self.assertEqual(str(1000 + offset), container[0]) + self.assertIn("rally_container_", container[1]) + + for index, tenant_id in enumerate(sorted(context["tenants"]), start=1): + containers = context["tenants"][tenant_id]["containers"] + self.assertEqual(containers_per_tenant, len(containers)) + for container in containers: + self.assertEqual("u%d" % index, container["user"]["id"]) + self.assertEqual("e%d" % index, container["user"]["endpoint"]) + self.assertIn("rally_container_", container["container"]) + self.assertEqual(0, len(container["objects"])) + + @mock.patch("rally.osclients.Clients") + def test__create_objects(self, mock_clients): + tenants = 2 + containers_per_tenant = 1 + objects_per_container = 5 + context = { + "task": {"uuid": "id123"}, + "tenants": { + "1001": { + "name": "t1_name", + "containers": [ + {"user": { + "id": "u1", "tenant_id": "1001", + "endpoint": "e1"}, + "container": "c1", + "objects": []} + ] + }, + "1002": { + "name": "t2_name", + "containers": [ + {"user": { + "id": "u2", "tenant_id": "1002", + "endpoint": "e2"}, + "container": "c2", + "objects": []} + ] + } + } + } + + mixin = utils.SwiftObjectMixin() + objects_list = mixin._create_objects(context, objects_per_container, + 1024, 25) + + self.assertEqual( + tenants * containers_per_tenant * objects_per_container, + len(objects_list)) + chunk = containers_per_tenant * objects_per_container + for index, obj in enumerate(sorted(objects_list)): + offset = int(index / chunk) + 1 + self.assertEqual(str(1000 + offset), obj[0]) + self.assertEqual("c%d" % offset, obj[1]) + self.assertIn("rally_object_", obj[2]) + + for tenant_id in context["tenants"]: + for container in context["tenants"][tenant_id]["containers"]: + self.assertEqual(objects_per_container, + len(container["objects"])) + for obj in container["objects"]: + self.assertIn("rally_object_", obj) + + @mock.patch("rally.osclients.Clients") + def test__delete_containers(self, mock_clients): + context = { + "task": {"uuid": "id123"}, + "tenants": { + "1001": { + "name": "t1_name", + "containers": [ + {"user": { + "id": "u1", "tenant_id": "1001", + "endpoint": "e1"}, + "container": "c1", + "objects": []} + ] + }, + "1002": { + "name": "t2_name", + "containers": [ + {"user": { + "id": "u2", "tenant_id": "1002", + "endpoint": "e2"}, + "container": "c2", + "objects": []} + ] + } + } + } + + mixin = utils.SwiftObjectMixin() + mixin._delete_containers(context, 1) + + mock_swift = mock_clients.return_value.swift.return_value + expected_containers = ["c1", "c2"] + mock_swift.delete_container.assert_has_calls( + [mock.call(con) for con in expected_containers], any_order=True) + + for tenant_id in context["tenants"]: + self.assertEqual(0, + len(context["tenants"][tenant_id]["containers"])) + + @mock.patch("rally.osclients.Clients") + def test__delete_objects(self, mock_clients): + context = { + "task": {"uuid": "id123"}, + "tenants": { + "1001": { + "name": "t1_name", + "containers": [ + {"user": { + "id": "u1", "tenant_id": "1001", + "endpoint": "e1"}, + "container": "c1", + "objects": ["o1", "o2", "o3"]} + ] + }, + "1002": { + "name": "t2_name", + "containers": [ + {"user": { + "id": "u2", "tenant_id": "1002", + "endpoint": "e2"}, + "container": "c2", + "objects": ["o4", "o5", "o6"]} + ] + } + } + } + + mixin = utils.SwiftObjectMixin() + mixin._delete_objects(context, 1) + + mock_swift = mock_clients.return_value.swift.return_value + expected_objects = [("c1", "o1"), ("c1", "o2"), ("c1", "o3"), + ("c2", "o4"), ("c2", "o5"), ("c2", "o6")] + mock_swift.delete_object.assert_has_calls( + [mock.call(con, obj) for con, obj in expected_objects], + any_order=True) + + for tenant_id in context["tenants"]: + for container in context["tenants"][tenant_id]["containers"]: + self.assertEqual(0, len(container["objects"]))