Merge "[Swift] Add objects context class"

rally/plugins/openstack/context/swift/objects.py

@@ -0,0 +1,100 @@
+# Copyright 2015: Cisco Systems, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from rally.common.i18n import _
+from rally.common import log as logging
+from rally.common import utils as rutils
+from rally import consts
+from rally import exceptions
+from rally.plugins.openstack.context.swift import utils as swift_utils
+from rally.task import context
+
+LOG = logging.getLogger(__name__)
+
+
+@context.configure(name="swift_objects", order=360)
+class SwiftObjectGenerator(swift_utils.SwiftObjectMixin, context.Context):
+    CONFIG_SCHEMA = {
+        "type": "object",
+        "$schema": consts.JSON_SCHEMA,
+        "properties": {
+            "containers_per_tenant": {
+                "type": "integer",
+                "minimum": 1
+            },
+            "objects_per_container": {
+                "type": "integer",
+                "minimum": 1
+            },
+            "object_size": {
+                "type": "integer",
+                "minimum": 1
+            },
+            "resource_management_workers": {
+                "type": "integer",
+                "minimum": 1
+            }
+        },
+        "additionalProperties": False
+    }
+
+    DEFAULT_CONFIG = {
+        "containers_per_tenant": 1,
+        "objects_per_container": 1,
+        "object_size": 1024,
+        "resource_management_workers": 30
+    }
+
+    @rutils.log_task_wrapper(LOG.info, _("Enter context: `swift_objects`"))
+    def setup(self):
+        """Create containers and objects, using the broker pattern."""
+        threads = self.config["resource_management_workers"]
+
+        containers_per_tenant = self.config["containers_per_tenant"]
+        containers_num = len(self.context["tenants"]) * containers_per_tenant
+        LOG.debug("Creating %d containers using %d threads." % (containers_num,
+                                                                threads))
+        containers_count = len(self._create_containers(self.context,
+                                                       containers_per_tenant,
+                                                       threads))
+        if containers_count != containers_num:
+            raise exceptions.ContextSetupFailure(
+                ctx_name=self.get_name(),
+                msg=_("Failed to create the requested number of containers, "
+                      "expected %(expected)s but got %(actual)s.")
+                % {"expected": containers_num, "actual": containers_count})
+
+        objects_per_container = self.config["objects_per_container"]
+        objects_num = containers_num * objects_per_container
+        LOG.debug("Creating %d objects using %d threads." % (objects_num,
+                                                             threads))
+        objects_count = len(self._create_objects(self.context,
+                                                 objects_per_container,
+                                                 self.config["object_size"],
+                                                 threads))
+        if objects_count != objects_num:
+            raise exceptions.ContextSetupFailure(
+                ctx_name=self.get_name(),
+                msg=_("Failed to create the requested number of objects, "
+                      "expected %(expected)s but got %(actual)s.")
+                % {"expected": objects_num, "actual": objects_count})
+
+    @rutils.log_task_wrapper(LOG.info, _("Exit context: `swift_objects`"))
+    def cleanup(self):
+        """Delete containers and objects, using the broker pattern."""
+        threads = self.config["resource_management_workers"]
+
+        self._delete_objects(self.context, threads)
+        self._delete_containers(self.context, threads)

rally/plugins/openstack/context/swift/utils.py

@@ -0,0 +1,148 @@
+# Copyright 2015: Cisco Systems, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import tempfile
+
+from rally.common import broker
+from rally.common import utils as rutils
+from rally.plugins.openstack.scenarios.swift import utils as swift_utils
+
+
+class SwiftObjectMixin(object):
+    """Mix-in method for Swift Object Context."""
+
+    def _create_containers(self, context, containers_per_tenant, threads):
+        """Create containers and store results in Rally context.
+
+        :param context: dict, Rally context environment
+        :param containers_per_tenant: int, number of containers to create
+                                      per tenant
+        :param threads: int, number of threads to use for broker pattern
+
+        :returns: list of tuples containing (account, container)
+        """
+        containers = []
+
+        def publish(queue):
+            for user, tenant_id in (rutils.iterate_per_tenants(
+                    context.get("users", []))):
+                context["tenants"][tenant_id]["containers"] = []
+                for i in range(containers_per_tenant):
+                    args = (user, context["tenants"][tenant_id]["containers"])
+                    queue.append(args)
+
+        def consume(cache, args):
+            user, tenant_containers = args
+            if user["id"] not in cache:
+                cache[user["id"]] = swift_utils.SwiftScenario({"user": user})
+            container_name = cache[user["id"]]._create_container()
+            tenant_containers.append({"user": user,
+                                      "container": container_name,
+                                      "objects": []})
+            containers.append((user["tenant_id"], container_name))
+
+        broker.run(publish, consume, threads)
+
+        return containers
+
+    def _create_objects(self, context, objects_per_container, object_size,
+                        threads):
+        """Create objects and store results in Rally context.
+
+        :param context: dict, Rally context environment
+        :param objects_per_container: int, number of objects to create
+                                      per container
+        :param object_size: int, size of created swift objects in byte
+        :param threads: int, number of threads to use for broker pattern
+
+        :returns: list of tuples containing (account, container, object)
+        """
+        objects = []
+
+        with tempfile.TemporaryFile() as dummy_file:
+            # set dummy file to specified object size
+            dummy_file.truncate(object_size)
+
+            def publish(queue):
+                for tenant_id in context["tenants"]:
+                    containers = context["tenants"][tenant_id]["containers"]
+                    for container in containers:
+                        for i in range(objects_per_container):
+                            queue.append(container)
+
+            def consume(cache, container):
+                user = container["user"]
+                if user["id"] not in cache:
+                    cache[user["id"]] = swift_utils.SwiftScenario(
+                        {"user": user})
+                dummy_file.seek(0)
+                object_name = cache[user["id"]]._upload_object(
+                    container["container"],
+                    dummy_file)[1]
+                container["objects"].append(object_name)
+                objects.append((user["tenant_id"], container["container"],
+                                object_name))
+
+            broker.run(publish, consume, threads)
+
+        return objects
+
+    def _delete_containers(self, context, threads):
+        """Delete containers created by Swift context and update Rally context.
+
+        :param context: dict, Rally context environment
+        :param threads: int, number of threads to use for broker pattern
+        """
+        def publish(queue):
+            for tenant_id in context["tenants"]:
+                containers = context["tenants"][tenant_id]["containers"]
+                for container in containers[:]:
+                    args = container, containers
+                    queue.append(args)
+
+        def consume(cache, args):
+            container, tenant_containers = args
+            user = container["user"]
+            if user["id"] not in cache:
+                cache[user["id"]] = swift_utils.SwiftScenario({"user": user})
+            cache[user["id"]]._delete_container(container["container"])
+            tenant_containers.remove(container)
+
+        broker.run(publish, consume, threads)
+
+    def _delete_objects(self, context, threads):
+        """Delete objects created by Swift context and update Rally context.
+
+        :param context: dict, Rally context environment
+        :param threads: int, number of threads to use for broker pattern
+        """
+        def publish(queue):
+            for tenant_id in context["tenants"]:
+                containers = context["tenants"][tenant_id]["containers"]
+                for container in containers:
+                    for object_name in container["objects"][:]:
+                        args = object_name, container
+                        queue.append(args)
+
+        def consume(cache, args):
+            object_name, container = args
+            user = container["user"]
+            if user["id"] not in cache:
+                cache[user["id"]] = swift_utils.SwiftScenario({"user": user})
+            cache[user["id"]]._delete_object(container["container"],
+                                             object_name)
+            container["objects"].remove(object_name)
+
+        broker.run(publish, consume, threads)

tests/unit/plugins/openstack/context/swift/test_objects.py

@@ -0,0 +1,204 @@
+# Copyright 2015: Cisco Systems, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+
+from rally import exceptions
+from rally.plugins.openstack.context.swift import objects
+from tests.unit import test
+
+
+class SwiftObjectGeneratorTestCase(test.TestCase):
+
+    @mock.patch("rally.osclients.Clients")
+    def test_setup(self, mock_clients):
+        containers_per_tenant = 2
+        objects_per_container = 7
+        context = {
+            "config": {
+                "swift_objects": {
+                    "containers_per_tenant": containers_per_tenant,
+                    "objects_per_container": objects_per_container,
+                    "object_size": 1024,
+                    "resource_management_workers": 10
+                }
+            },
+            "task": {"uuid": "id123"},
+            "tenants": {
+                "t1": {"name": "t1_name"},
+                "t2": {"name": "t2_name"}
+            },
+            "users": [
+                {"id": "u1", "tenant_id": "t1", "endpoint": "e1"},
+                {"id": "u2", "tenant_id": "t2", "endpoint": "e2"}
+            ]
+        }
+
+        objects_ctx = objects.SwiftObjectGenerator(context)
+        objects_ctx.setup()
+
+        for tenant_id in context["tenants"]:
+            containers = context["tenants"][tenant_id]["containers"]
+            self.assertEqual(containers_per_tenant, len(containers))
+            for container in containers:
+                self.assertIn("rally_container_", container["container"])
+                self.assertEqual(objects_per_container,
+                                 len(container["objects"]))
+                for obj in container["objects"]:
+                    self.assertIn("rally_object_", obj)
+
+    @mock.patch("rally.osclients.Clients")
+    @mock.patch("rally.plugins.openstack.context.swift.utils."
+                "swift_utils.SwiftScenario")
+    def test_cleanup(self, mock_swift_scenario, mock_clients):
+        context = {
+            "config": {
+                "swift_objects": {
+                    "resource_management_workers": 1
+                }
+            },
+            "task": {"uuid": "id123"},
+            "tenants": {
+                "t1": {
+                    "name": "t1_name",
+                    "containers": [
+                        {"user": {"id": "u1", "tenant_id": "t1",
+                                  "endpoint": "e1"},
+                         "container": "c1",
+                         "objects": ["o1", "o2", "o3"]}
+                    ]
+                },
+                "t2": {
+                    "name": "t2_name",
+                    "containers": [
+                        {"user": {"id": "u2", "tenant_id": "t2",
+                                  "endpoint": "e2"},
+                         "container": "c2",
+                         "objects": ["o4", "o5", "o6"]}
+                    ]
+                }
+            }
+        }
+
+        objects_ctx = objects.SwiftObjectGenerator(context)
+        objects_ctx.cleanup()
+
+        expected_containers = ["c1", "c2"]
+        mock_swift_scenario.return_value._delete_container.assert_has_calls(
+            [mock.call(con) for con in expected_containers], any_order=True)
+
+        expected_objects = [("c1", "o1"), ("c1", "o2"), ("c1", "o3"),
+                            ("c2", "o4"), ("c2", "o5"), ("c2", "o6")]
+        mock_swift_scenario.return_value._delete_object.assert_has_calls(
+            [mock.call(con, obj) for con, obj in expected_objects],
+            any_order=True)
+
+        for tenant_id in context["tenants"]:
+            self.assertEqual(0,
+                             len(context["tenants"][tenant_id]["containers"]))
+
+    @mock.patch("rally.osclients.Clients")
+    def test_setup_failure_clients_put_container(self, mock_clients):
+        context = {
+            "config": {
+                "swift_objects": {
+                    "containers_per_tenant": 2,
+                    "object_size": 10,
+                    "resource_management_workers": 5
+                }
+            },
+            "task": {"uuid": "id123"},
+            "tenants": {
+                "t1": {"name": "t1_name"},
+                "t2": {"name": "t2_name"}
+            },
+            "users": [
+                {"id": "u1", "tenant_id": "t1", "endpoint": "e1"},
+                {"id": "u2", "tenant_id": "t2", "endpoint": "e2"}
+            ]
+        }
+        mock_swift = mock_clients.return_value.swift.return_value
+        mock_swift.put_container.side_effect = [Exception, True,
+                                                Exception, Exception]
+        objects_ctx = objects.SwiftObjectGenerator(context)
+        self.assertRaisesRegexp(exceptions.ContextSetupFailure,
+                                "containers, expected 4 but got 1",
+                                objects_ctx.setup)
+
+    @mock.patch("rally.osclients.Clients")
+    def test_setup_failure_clients_put_object(self, mock_clients):
+        context = {
+            "task": {"uuid": "id123"},
+            "tenants": {
+                "t1": {"name": "t1_name"},
+                "t2": {"name": "t2_name"}
+            },
+            "users": [
+                {"id": "u1", "tenant_id": "t1", "endpoint": "e1"},
+                {"id": "u2", "tenant_id": "t2", "endpoint": "e2"}
+            ]
+        }
+        mock_swift = mock_clients.return_value.swift.return_value
+        mock_swift.put_object.side_effect = [Exception, True]
+        objects_ctx = objects.SwiftObjectGenerator(context)
+        self.assertRaisesRegexp(exceptions.ContextSetupFailure,
+                                "objects, expected 2 but got 1",
+                                objects_ctx.setup)
+
+    @mock.patch("rally.osclients.Clients")
+    def test_cleanup_failure_clients_delete_container(self, mock_clients):
+        context = {
+            "task": {"uuid": "id123"},
+            "tenants": {
+                "t1": {
+                    "name": "t1_name",
+                    "containers": [
+                        {"user": {"id": "u1", "tenant_id": "t1",
+                                  "endpoint": "e1"},
+                         "container": "coooon",
+                         "objects": []}] * 3
+                }
+            }
+        }
+        mock_swift = mock_clients.return_value.swift.return_value
+        mock_swift.delete_container.side_effect = [True, True, Exception]
+        objects_ctx = objects.SwiftObjectGenerator(context)
+        objects_ctx.cleanup()
+        self.assertEqual(1, len(context["tenants"]["t1"]["containers"]))
+
+    @mock.patch("rally.osclients.Clients")
+    def test_cleanup_failure_clients_delete_object(self, mock_clients):
+        context = {
+            "task": {"uuid": "id123"},
+            "tenants": {
+                "t1": {
+                    "name": "t1_name",
+                    "containers": [
+                        {"user": {"id": "u1", "tenant_id": "t1",
+                                  "endpoint": "e1"},
+                         "container": "c1",
+                         "objects": ["oooo"] * 3}
+                    ]
+                }
+            }
+        }
+        mock_swift = mock_clients.return_value.swift.return_value
+        mock_swift.delete_object.side_effect = [True, Exception, True]
+        objects_ctx = objects.SwiftObjectGenerator(context)
+        objects_ctx._delete_containers = mock.MagicMock()
+        objects_ctx.cleanup()
+        self.assertEqual(
+            1, sum([len(container["objects"])
+                    for container in context["tenants"]["t1"]["containers"]]))

tests/unit/plugins/openstack/context/swift/test_utils.py

@@ -0,0 +1,191 @@
+# Copyright 2015: Cisco Systems, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+
+from rally.plugins.openstack.context.swift import utils
+from tests.unit import test
+
+
+class SwiftObjectMixinTestCase(test.TestCase):
+
+    @mock.patch("rally.osclients.Clients")
+    def test__create_containers(self, mock_clients):
+        tenants = 2
+        containers_per_tenant = 2
+        context = {
+            "task": {"uuid": "id123"},
+            "tenants": {
+                "1001": {"name": "t1_name"},
+                "1002": {"name": "t2_name"}
+            },
+            "users": [
+                {"id": "u1", "tenant_id": "1001", "endpoint": "e1"},
+                {"id": "u2", "tenant_id": "1002", "endpoint": "e2"}
+            ]
+        }
+
+        mixin = utils.SwiftObjectMixin()
+        containers = mixin._create_containers(context, containers_per_tenant,
+                                              15)
+
+        self.assertEqual(tenants * containers_per_tenant, len(containers))
+        for index, container in enumerate(sorted(containers)):
+            offset = int(index / containers_per_tenant) + 1
+            self.assertEqual(str(1000 + offset), container[0])
+            self.assertIn("rally_container_", container[1])
+
+        for index, tenant_id in enumerate(sorted(context["tenants"]), start=1):
+            containers = context["tenants"][tenant_id]["containers"]
+            self.assertEqual(containers_per_tenant, len(containers))
+            for container in containers:
+                self.assertEqual("u%d" % index, container["user"]["id"])
+                self.assertEqual("e%d" % index, container["user"]["endpoint"])
+                self.assertIn("rally_container_", container["container"])
+                self.assertEqual(0, len(container["objects"]))
+
+    @mock.patch("rally.osclients.Clients")
+    def test__create_objects(self, mock_clients):
+        tenants = 2
+        containers_per_tenant = 1
+        objects_per_container = 5
+        context = {
+            "task": {"uuid": "id123"},
+            "tenants": {
+                "1001": {
+                    "name": "t1_name",
+                    "containers": [
+                        {"user": {
+                            "id": "u1", "tenant_id": "1001",
+                            "endpoint": "e1"},
+                         "container": "c1",
+                         "objects": []}
+                    ]
+                },
+                "1002": {
+                    "name": "t2_name",
+                    "containers": [
+                        {"user": {
+                            "id": "u2", "tenant_id": "1002",
+                            "endpoint": "e2"},
+                         "container": "c2",
+                         "objects": []}
+                    ]
+                }
+            }
+        }
+
+        mixin = utils.SwiftObjectMixin()
+        objects_list = mixin._create_objects(context, objects_per_container,
+                                             1024, 25)
+
+        self.assertEqual(
+            tenants * containers_per_tenant * objects_per_container,
+            len(objects_list))
+        chunk = containers_per_tenant * objects_per_container
+        for index, obj in enumerate(sorted(objects_list)):
+            offset = int(index / chunk) + 1
+            self.assertEqual(str(1000 + offset), obj[0])
+            self.assertEqual("c%d" % offset, obj[1])
+            self.assertIn("rally_object_", obj[2])
+
+        for tenant_id in context["tenants"]:
+            for container in context["tenants"][tenant_id]["containers"]:
+                self.assertEqual(objects_per_container,
+                                 len(container["objects"]))
+                for obj in container["objects"]:
+                    self.assertIn("rally_object_", obj)
+
+    @mock.patch("rally.osclients.Clients")
+    def test__delete_containers(self, mock_clients):
+        context = {
+            "task": {"uuid": "id123"},
+            "tenants": {
+                "1001": {
+                    "name": "t1_name",
+                    "containers": [
+                        {"user": {
+                            "id": "u1", "tenant_id": "1001",
+                            "endpoint": "e1"},
+                         "container": "c1",
+                         "objects": []}
+                    ]
+                },
+                "1002": {
+                    "name": "t2_name",
+                    "containers": [
+                        {"user": {
+                            "id": "u2", "tenant_id": "1002",
+                            "endpoint": "e2"},
+                         "container": "c2",
+                         "objects": []}
+                    ]
+                }
+            }
+        }
+
+        mixin = utils.SwiftObjectMixin()
+        mixin._delete_containers(context, 1)
+
+        mock_swift = mock_clients.return_value.swift.return_value
+        expected_containers = ["c1", "c2"]
+        mock_swift.delete_container.assert_has_calls(
+            [mock.call(con) for con in expected_containers], any_order=True)
+
+        for tenant_id in context["tenants"]:
+            self.assertEqual(0,
+                             len(context["tenants"][tenant_id]["containers"]))
+
+    @mock.patch("rally.osclients.Clients")
+    def test__delete_objects(self, mock_clients):
+        context = {
+            "task": {"uuid": "id123"},
+            "tenants": {
+                "1001": {
+                    "name": "t1_name",
+                    "containers": [
+                        {"user": {
+                            "id": "u1", "tenant_id": "1001",
+                            "endpoint": "e1"},
+                         "container": "c1",
+                         "objects": ["o1", "o2", "o3"]}
+                    ]
+                },
+                "1002": {
+                    "name": "t2_name",
+                    "containers": [
+                        {"user": {
+                            "id": "u2", "tenant_id": "1002",
+                            "endpoint": "e2"},
+                         "container": "c2",
+                         "objects": ["o4", "o5", "o6"]}
+                    ]
+                }
+            }
+        }
+
+        mixin = utils.SwiftObjectMixin()
+        mixin._delete_objects(context, 1)
+
+        mock_swift = mock_clients.return_value.swift.return_value
+        expected_objects = [("c1", "o1"), ("c1", "o2"), ("c1", "o3"),
+                            ("c2", "o4"), ("c2", "o5"), ("c2", "o6")]
+        mock_swift.delete_object.assert_has_calls(
+            [mock.call(con, obj) for con, obj in expected_objects],
+            any_order=True)
+
+        for tenant_id in context["tenants"]:
+            for container in context["tenants"][tenant_id]["containers"]:
+                self.assertEqual(0, len(container["objects"]))