[Swift] Add objects context class

This patch adds new context for creating swift objects using broker
pattern.  It will create large number of containers and objects
concurrently before benchmark task like download or list objects and
delete it after task finishes.

Blueprint benchmark-scenarios-for-swift

Change-Id: I0c8623fbe56401171244542ad8188d20f8848e8c

rally/plugins/openstack/context/swift/objects.py

@@ -0,0 +1,100 @@
+# Copyright 2015: Cisco Systems, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from rally.common.i18n import _
+from rally.common import log as logging
+from rally.common import utils as rutils
+from rally import consts
+from rally import exceptions
+from rally.plugins.openstack.context.swift import utils as swift_utils
+from rally.task import context
+
+LOG = logging.getLogger(__name__)
+
+
+@context.configure(name="swift_objects", order=360)
+class SwiftObjectGenerator(swift_utils.SwiftObjectMixin, context.Context):
+    CONFIG_SCHEMA = {
+        "type": "object",
+        "$schema": consts.JSON_SCHEMA,
+        "properties": {
+            "containers_per_tenant": {
+                "type": "integer",
+                "minimum": 1
+            },
+            "objects_per_container": {
+                "type": "integer",
+                "minimum": 1
+            },
+            "object_size": {
+                "type": "integer",
+                "minimum": 1
+            },
+            "resource_management_workers": {
+                "type": "integer",
+                "minimum": 1
+            }
+        },
+        "additionalProperties": False
+    }
+
+    DEFAULT_CONFIG = {
+        "containers_per_tenant": 1,
+        "objects_per_container": 1,
+        "object_size": 1024,
+        "resource_management_workers": 30
+    }
+
+    @rutils.log_task_wrapper(LOG.info, _("Enter context: `swift_objects`"))
+    def setup(self):
+        """Create containers and objects, using the broker pattern."""
+        threads = self.config["resource_management_workers"]
+
+        containers_per_tenant = self.config["containers_per_tenant"]
+        containers_num = len(self.context["tenants"]) * containers_per_tenant
+        LOG.debug("Creating %d containers using %d threads." % (containers_num,
+                                                                threads))
+        containers_count = len(self._create_containers(self.context,
+                                                       containers_per_tenant,
+                                                       threads))
+        if containers_count != containers_num:
+            raise exceptions.ContextSetupFailure(
+                ctx_name=self.get_name(),
+                msg=_("Failed to create the requested number of containers, "
+                      "expected %(expected)s but got %(actual)s.")
+                % {"expected": containers_num, "actual": containers_count})
+
+        objects_per_container = self.config["objects_per_container"]
+        objects_num = containers_num * objects_per_container
+        LOG.debug("Creating %d objects using %d threads." % (objects_num,
+                                                             threads))
+        objects_count = len(self._create_objects(self.context,
+                                                 objects_per_container,
+                                                 self.config["object_size"],
+                                                 threads))
+        if objects_count != objects_num:
+            raise exceptions.ContextSetupFailure(
+                ctx_name=self.get_name(),
+                msg=_("Failed to create the requested number of objects, "
+                      "expected %(expected)s but got %(actual)s.")
+                % {"expected": objects_num, "actual": objects_count})
+
+    @rutils.log_task_wrapper(LOG.info, _("Exit context: `swift_objects`"))
+    def cleanup(self):
+        """Delete containers and objects, using the broker pattern."""
+        threads = self.config["resource_management_workers"]
+
+        self._delete_objects(self.context, threads)
+        self._delete_containers(self.context, threads)

rally/plugins/openstack/context/swift/utils.py

@@ -0,0 +1,148 @@
+# Copyright 2015: Cisco Systems, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import tempfile
+
+from rally.common import broker
+from rally.common import utils as rutils
+from rally.plugins.openstack.scenarios.swift import utils as swift_utils
+
+
+class SwiftObjectMixin(object):
+    """Mix-in method for Swift Object Context."""
+
+    def _create_containers(self, context, containers_per_tenant, threads):
+        """Create containers and store results in Rally context.
+
+        :param context: dict, Rally context environment
+        :param containers_per_tenant: int, number of containers to create
+                                      per tenant
+        :param threads: int, number of threads to use for broker pattern
+
+        :returns: list of tuples containing (account, container)
+        """
+        containers = []
+
+        def publish(queue):
+            for user, tenant_id in (rutils.iterate_per_tenants(
+                    context.get("users", []))):
+                context["tenants"][tenant_id]["containers"] = []
+                for i in range(containers_per_tenant):
+                    args = (user, context["tenants"][tenant_id]["containers"])
+                    queue.append(args)
+
+        def consume(cache, args):
+            user, tenant_containers = args
+            if user["id"] not in cache:
+                cache[user["id"]] = swift_utils.SwiftScenario({"user": user})
+            container_name = cache[user["id"]]._create_container()
+            tenant_containers.append({"user": user,
+                                      "container": container_name,
+                                      "objects": []})
+            containers.append((user["tenant_id"], container_name))
+
+        broker.run(publish, consume, threads)
+
+        return containers
+
+    def _create_objects(self, context, objects_per_container, object_size,
+                        threads):
+        """Create objects and store results in Rally context.
+
+        :param context: dict, Rally context environment
+        :param objects_per_container: int, number of objects to create
+                                      per container
+        :param object_size: int, size of created swift objects in byte
+        :param threads: int, number of threads to use for broker pattern
+
+        :returns: list of tuples containing (account, container, object)
+        """
+        objects = []
+
+        with tempfile.TemporaryFile() as dummy_file:
+            # set dummy file to specified object size
+            dummy_file.truncate(object_size)
+
+            def publish(queue):
+                for tenant_id in context["tenants"]:
+                    containers = context["tenants"][tenant_id]["containers"]
+                    for container in containers:
+                        for i in range(objects_per_container):
+                            queue.append(container)
+
+            def consume(cache, container):
+                user = container["user"]
+                if user["id"] not in cache:
+                    cache[user["id"]] = swift_utils.SwiftScenario(
+                        {"user": user})
+                dummy_file.seek(0)
+                object_name = cache[user["id"]]._upload_object(
+                    container["container"],
+                    dummy_file)[1]
+                container["objects"].append(object_name)
+                objects.append((user["tenant_id"], container["container"],
+                                object_name))
+
+            broker.run(publish, consume, threads)
+
+        return objects
+
+    def _delete_containers(self, context, threads):
+        """Delete containers created by Swift context and update Rally context.
+
+        :param context: dict, Rally context environment
+        :param threads: int, number of threads to use for broker pattern
+        """
+        def publish(queue):
+            for tenant_id in context["tenants"]:
+                containers = context["tenants"][tenant_id]["containers"]
+                for container in containers[:]:
+                    args = container, containers
+                    queue.append(args)
+
+        def consume(cache, args):
+            container, tenant_containers = args
+            user = container["user"]
+            if user["id"] not in cache:
+                cache[user["id"]] = swift_utils.SwiftScenario({"user": user})
+            cache[user["id"]]._delete_container(container["container"])
+            tenant_containers.remove(container)
+
+        broker.run(publish, consume, threads)
+
+    def _delete_objects(self, context, threads):
+        """Delete objects created by Swift context and update Rally context.
+
+        :param context: dict, Rally context environment
+        :param threads: int, number of threads to use for broker pattern
+        """
+        def publish(queue):
+            for tenant_id in context["tenants"]:
+                containers = context["tenants"][tenant_id]["containers"]
+                for container in containers:
+                    for object_name in container["objects"][:]:
+                        args = object_name, container
+                        queue.append(args)
+
+        def consume(cache, args):
+            object_name, container = args
+            user = container["user"]
+            if user["id"] not in cache:
+                cache[user["id"]] = swift_utils.SwiftScenario({"user": user})
+            cache[user["id"]]._delete_object(container["container"],
+                                             object_name)
+            container["objects"].remove(object_name)
+
+        broker.run(publish, consume, threads)

tests/unit/plugins/openstack/context/swift/test_objects.py

@@ -0,0 +1,204 @@
+# Copyright 2015: Cisco Systems, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+
+from rally import exceptions
+from rally.plugins.openstack.context.swift import objects
+from tests.unit import test
+
+
+class SwiftObjectGeneratorTestCase(test.TestCase):
+
+    @mock.patch("rally.osclients.Clients")
+    def test_setup(self, mock_clients):
+        containers_per_tenant = 2
+        objects_per_container = 7
+        context = {
+            "config": {
+                "swift_objects": {
+                    "containers_per_tenant": containers_per_tenant,
+                    "objects_per_container": objects_per_container,
+                    "object_size": 1024,
+                    "resource_management_workers": 10
+                }
+            },
+            "task": {"uuid": "id123"},
+            "tenants": {
+                "t1": {"name": "t1_name"},
+                "t2": {"name": "t2_name"}
+            },
+            "users": [
+                {"id": "u1", "tenant_id": "t1", "endpoint": "e1"},
+                {"id": "u2", "tenant_id": "t2", "endpoint": "e2"}
+            ]
+        }
+
+        objects_ctx = objects.SwiftObjectGenerator(context)
+        objects_ctx.setup()
+
+        for tenant_id in context["tenants"]:
+            containers = context["tenants"][tenant_id]["containers"]
+            self.assertEqual(containers_per_tenant, len(containers))
+            for container in containers:
+                self.assertIn("rally_container_", container["container"])
+                self.assertEqual(objects_per_container,
+                                 len(container["objects"]))
+                for obj in container["objects"]:
+                    self.assertIn("rally_object_", obj)
+
+    @mock.patch("rally.osclients.Clients")
+    @mock.patch("rally.plugins.openstack.context.swift.utils."
+                "swift_utils.SwiftScenario")
+    def test_cleanup(self, mock_swift_scenario, mock_clients):
+        context = {
+            "config": {
+                "swift_objects": {
+                    "resource_management_workers": 1
+                }
+            },
+            "task": {"uuid": "id123"},
+            "tenants": {
+                "t1": {
+                    "name": "t1_name",
+                    "containers": [
+                        {"user": {"id": "u1", "tenant_id": "t1",
+                                  "endpoint": "e1"},
+                         "container": "c1",
+                         "objects": ["o1", "o2", "o3"]}
+                    ]
+                },
+                "t2": {
+                    "name": "t2_name",
+                    "containers": [
+                        {"user": {"id": "u2", "tenant_id": "t2",
+                                  "endpoint": "e2"},
+                         "container": "c2",
+                         "objects": ["o4", "o5", "o6"]}
+                    ]
+                }
+            }
+        }
+
+        objects_ctx = objects.SwiftObjectGenerator(context)
+        objects_ctx.cleanup()
+
+        expected_containers = ["c1", "c2"]
+        mock_swift_scenario.return_value._delete_container.assert_has_calls(
+            [mock.call(con) for con in expected_containers], any_order=True)
+
+        expected_objects = [("c1", "o1"), ("c1", "o2"), ("c1", "o3"),
+                            ("c2", "o4"), ("c2", "o5"), ("c2", "o6")]
+        mock_swift_scenario.return_value._delete_object.assert_has_calls(
+            [mock.call(con, obj) for con, obj in expected_objects],
+            any_order=True)
+
+        for tenant_id in context["tenants"]:
+            self.assertEqual(0,
+                             len(context["tenants"][tenant_id]["containers"]))
+
+    @mock.patch("rally.osclients.Clients")
+    def test_setup_failure_clients_put_container(self, mock_clients):
+        context = {
+            "config": {
+                "swift_objects": {
+                    "containers_per_tenant": 2,
+                    "object_size": 10,
+                    "resource_management_workers": 5
+                }
+            },
+            "task": {"uuid": "id123"},
+            "tenants": {
+                "t1": {"name": "t1_name"},
+                "t2": {"name": "t2_name"}
+            },
+            "users": [
+                {"id": "u1", "tenant_id": "t1", "endpoint": "e1"},
+                {"id": "u2", "tenant_id": "t2", "endpoint": "e2"}
+            ]
+        }
+        mock_swift = mock_clients.return_value.swift.return_value
+        mock_swift.put_container.side_effect = [Exception, True,
+                                                Exception, Exception]
+        objects_ctx = objects.SwiftObjectGenerator(context)
+        self.assertRaisesRegexp(exceptions.ContextSetupFailure,
+                                "containers, expected 4 but got 1",
+                                objects_ctx.setup)
+
+    @mock.patch("rally.osclients.Clients")
+    def test_setup_failure_clients_put_object(self, mock_clients):
+        context = {
+            "task": {"uuid": "id123"},
+            "tenants": {
+                "t1": {"name": "t1_name"},
+                "t2": {"name": "t2_name"}
+            },
+            "users": [
+                {"id": "u1", "tenant_id": "t1", "endpoint": "e1"},
+                {"id": "u2", "tenant_id": "t2", "endpoint": "e2"}
+            ]
+        }
+        mock_swift = mock_clients.return_value.swift.return_value
+        mock_swift.put_object.side_effect = [Exception, True]
+        objects_ctx = objects.SwiftObjectGenerator(context)
+        self.assertRaisesRegexp(exceptions.ContextSetupFailure,
+                                "objects, expected 2 but got 1",
+                                objects_ctx.setup)
+
+    @mock.patch("rally.osclients.Clients")
+    def test_cleanup_failure_clients_delete_container(self, mock_clients):
+        context = {
+            "task": {"uuid": "id123"},
+            "tenants": {
+                "t1": {
+                    "name": "t1_name",
+                    "containers": [
+                        {"user": {"id": "u1", "tenant_id": "t1",
+                                  "endpoint": "e1"},
+                         "container": "coooon",
+                         "objects": []}] * 3
+                }
+            }
+        }
+        mock_swift = mock_clients.return_value.swift.return_value
+        mock_swift.delete_container.side_effect = [True, True, Exception]
+        objects_ctx = objects.SwiftObjectGenerator(context)
+        objects_ctx.cleanup()
+        self.assertEqual(1, len(context["tenants"]["t1"]["containers"]))
+
+    @mock.patch("rally.osclients.Clients")
+    def test_cleanup_failure_clients_delete_object(self, mock_clients):
+        context = {
+            "task": {"uuid": "id123"},
+            "tenants": {
+                "t1": {
+                    "name": "t1_name",
+                    "containers": [
+                        {"user": {"id": "u1", "tenant_id": "t1",
+                                  "endpoint": "e1"},
+                         "container": "c1",
+                         "objects": ["oooo"] * 3}
+                    ]
+                }
+            }
+        }
+        mock_swift = mock_clients.return_value.swift.return_value
+        mock_swift.delete_object.side_effect = [True, Exception, True]
+        objects_ctx = objects.SwiftObjectGenerator(context)
+        objects_ctx._delete_containers = mock.MagicMock()
+        objects_ctx.cleanup()
+        self.assertEqual(
+            1, sum([len(container["objects"])
+                    for container in context["tenants"]["t1"]["containers"]]))

tests/unit/plugins/openstack/context/swift/test_utils.py

@@ -0,0 +1,191 @@
+# Copyright 2015: Cisco Systems, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+
+from rally.plugins.openstack.context.swift import utils
+from tests.unit import test
+
+
+class SwiftObjectMixinTestCase(test.TestCase):
+
+    @mock.patch("rally.osclients.Clients")
+    def test__create_containers(self, mock_clients):
+        tenants = 2
+        containers_per_tenant = 2
+        context = {
+            "task": {"uuid": "id123"},
+            "tenants": {
+                "1001": {"name": "t1_name"},
+                "1002": {"name": "t2_name"}
+            },
+            "users": [
+                {"id": "u1", "tenant_id": "1001", "endpoint": "e1"},
+                {"id": "u2", "tenant_id": "1002", "endpoint": "e2"}
+            ]
+        }
+
+        mixin = utils.SwiftObjectMixin()
+        containers = mixin._create_containers(context, containers_per_tenant,
+                                              15)
+
+        self.assertEqual(tenants * containers_per_tenant, len(containers))
+        for index, container in enumerate(sorted(containers)):
+            offset = int(index / containers_per_tenant) + 1
+            self.assertEqual(str(1000 + offset), container[0])
+            self.assertIn("rally_container_", container[1])
+
+        for index, tenant_id in enumerate(sorted(context["tenants"]), start=1):
+            containers = context["tenants"][tenant_id]["containers"]
+            self.assertEqual(containers_per_tenant, len(containers))
+            for container in containers:
+                self.assertEqual("u%d" % index, container["user"]["id"])
+                self.assertEqual("e%d" % index, container["user"]["endpoint"])
+                self.assertIn("rally_container_", container["container"])
+                self.assertEqual(0, len(container["objects"]))
+
+    @mock.patch("rally.osclients.Clients")
+    def test__create_objects(self, mock_clients):
+        tenants = 2
+        containers_per_tenant = 1
+        objects_per_container = 5
+        context = {
+            "task": {"uuid": "id123"},
+            "tenants": {
+                "1001": {
+                    "name": "t1_name",
+                    "containers": [
+                        {"user": {
+                            "id": "u1", "tenant_id": "1001",
+                            "endpoint": "e1"},
+                         "container": "c1",
+                         "objects": []}
+                    ]
+                },
+                "1002": {
+                    "name": "t2_name",
+                    "containers": [
+                        {"user": {
+                            "id": "u2", "tenant_id": "1002",
+                            "endpoint": "e2"},
+                         "container": "c2",
+                         "objects": []}
+                    ]
+                }
+            }
+        }
+
+        mixin = utils.SwiftObjectMixin()
+        objects_list = mixin._create_objects(context, objects_per_container,
+                                             1024, 25)
+
+        self.assertEqual(
+            tenants * containers_per_tenant * objects_per_container,
+            len(objects_list))
+        chunk = containers_per_tenant * objects_per_container
+        for index, obj in enumerate(sorted(objects_list)):
+            offset = int(index / chunk) + 1
+            self.assertEqual(str(1000 + offset), obj[0])
+            self.assertEqual("c%d" % offset, obj[1])
+            self.assertIn("rally_object_", obj[2])
+
+        for tenant_id in context["tenants"]:
+            for container in context["tenants"][tenant_id]["containers"]:
+                self.assertEqual(objects_per_container,
+                                 len(container["objects"]))
+                for obj in container["objects"]:
+                    self.assertIn("rally_object_", obj)
+
+    @mock.patch("rally.osclients.Clients")
+    def test__delete_containers(self, mock_clients):
+        context = {
+            "task": {"uuid": "id123"},
+            "tenants": {
+                "1001": {
+                    "name": "t1_name",
+                    "containers": [
+                        {"user": {
+                            "id": "u1", "tenant_id": "1001",
+                            "endpoint": "e1"},
+                         "container": "c1",
+                         "objects": []}
+                    ]
+                },
+                "1002": {
+                    "name": "t2_name",
+                    "containers": [
+                        {"user": {
+                            "id": "u2", "tenant_id": "1002",
+                            "endpoint": "e2"},
+                         "container": "c2",
+                         "objects": []}
+                    ]
+                }
+            }
+        }
+
+        mixin = utils.SwiftObjectMixin()
+        mixin._delete_containers(context, 1)
+
+        mock_swift = mock_clients.return_value.swift.return_value
+        expected_containers = ["c1", "c2"]
+        mock_swift.delete_container.assert_has_calls(
+            [mock.call(con) for con in expected_containers], any_order=True)
+
+        for tenant_id in context["tenants"]:
+            self.assertEqual(0,
+                             len(context["tenants"][tenant_id]["containers"]))
+
+    @mock.patch("rally.osclients.Clients")
+    def test__delete_objects(self, mock_clients):
+        context = {
+            "task": {"uuid": "id123"},
+            "tenants": {
+                "1001": {
+                    "name": "t1_name",
+                    "containers": [
+                        {"user": {
+                            "id": "u1", "tenant_id": "1001",
+                            "endpoint": "e1"},
+                         "container": "c1",
+                         "objects": ["o1", "o2", "o3"]}
+                    ]
+                },
+                "1002": {
+                    "name": "t2_name",
+                    "containers": [
+                        {"user": {
+                            "id": "u2", "tenant_id": "1002",
+                            "endpoint": "e2"},
+                         "container": "c2",
+                         "objects": ["o4", "o5", "o6"]}
+                    ]
+                }
+            }
+        }
+
+        mixin = utils.SwiftObjectMixin()
+        mixin._delete_objects(context, 1)
+
+        mock_swift = mock_clients.return_value.swift.return_value
+        expected_objects = [("c1", "o1"), ("c1", "o2"), ("c1", "o3"),
+                            ("c2", "o4"), ("c2", "o5"), ("c2", "o6")]
+        mock_swift.delete_object.assert_has_calls(
+            [mock.call(con, obj) for con, obj in expected_objects],
+            any_order=True)
+
+        for tenant_id in context["tenants"]:
+            for container in context["tenants"][tenant_id]["containers"]:
+                self.assertEqual(0, len(container["objects"]))