Remove existing_users context

There is no need to have two separate classes - users and existing_users.
Both of them are equal from task engine perspective, but require splitting
some logic based on deployment configuration.

This patch simplifies that things and merges "existing_users" context into
"users".

Co-Authored-By: Andrey Kurilin <andr.kurilin@gmail.com>

Change-Id: Ie8fe3b433686bbbd80197876662ccb87b972d5a3

rally/plugins/openstack/context/keystone/users.py

@@ -21,7 +21,9 @@ from oslo_config import cfg
 from rally.common import broker
 from rally.common.i18n import _
 from rally.common import logging
+from rally.common import objects
 from rally.common import utils as rutils
+from rally.common import validation
 from rally import consts
 from rally import exceptions
 from rally import osclients
@@ -29,7 +31,6 @@ from rally.plugins.openstack import credential
 from rally.plugins.openstack.services.identity import identity
 from rally.plugins.openstack.wrappers import network
 from rally.task import context
-from rally.task import validation
 
 from rally.common import opts
 opts.register()
@@ -45,7 +46,7 @@ PROJECT_DOMAIN_DESCR = "ID of domain in which projects will be created."
 USER_DOMAIN_DESCR = "ID of domain in which users will be created."
 
 
-@validation.add("required_platform", platform="openstack", admin=True)
+@validation.add("required_platform", platform="openstack", users=True)
 @context.configure(name="users", namespace="openstack", order=100)
 class UserGenerator(context.Context):
     """Context class for generating temporary users/tenants for benchmarks."""
@@ -53,58 +54,78 @@ class UserGenerator(context.Context):
     CONFIG_SCHEMA = {
         "type": "object",
         "$schema": consts.JSON_SCHEMA,
-        "properties": {
+        "oneOf": [
-            "tenants": {
+            {"description": "Create new temporary users and tenants.",
-                "type": "integer",
+             "properties": {
-                "minimum": 1,
+                "tenants": {
-                "description": "The number of tenants to create."
+                    "type": "integer",
-            },
+                    "minimum": 1,
-            "users_per_tenant": {
+                    "description": "The number of tenants to create."
-                "type": "integer",
+                },
-                "minimum": 1,
+                 "users_per_tenant": {
-                "description": "The number of users to create per one tenant."
+                     "type": "integer",
-            },
+                     "minimum": 1,
-            "resource_management_workers": {
+                     "description": "The number of users to create per one "
-                "type": "integer",
+                                    "tenant."},
-                "minimum": 1,
+                 "resource_management_workers": {
-                "description": RESOURCE_MANAGEMENT_WORKERS_DESCR,
+                     "type": "integer",
-
+                     "minimum": 1,
-            },
+                     "description": RESOURCE_MANAGEMENT_WORKERS_DESCR},
-            "project_domain": {
+                 "project_domain": {
-                "type": "string",
+                     "type": "string",
-                "description": PROJECT_DOMAIN_DESCR
+                     "description": PROJECT_DOMAIN_DESCR},
-            },
+                 "user_domain": {
-            "user_domain": {
+                     "type": "string",
-                "type": "string",
+                     "description": USER_DOMAIN_DESCR},
-                "description": USER_DOMAIN_DESCR
+                 "user_choice_method": {
-            },
+                     "$ref": "#/definitions/user_choice_method"}},
+             "additionalProperties": False},
+            # TODO(andreykurilin): add ability to specify users here.
+            {"description": "Use existing users and tenants.",
+             "properties": {
+                 "user_choice_method": {
+                     "$ref": "#/definitions/user_choice_method"}
+             },
+             "additionalProperties": False}
+        ],
+        "definitions": {
             "user_choice_method": {
                 "enum": ["random", "round_robin"],
                 "description": "The mode of balancing usage of users between "
-                               "scenario iterations."
+                               "scenario iterations."}
-            },
+
-        },
+        }
-        "additionalProperties": False
     }
 
-    DEFAULT_CONFIG = {
+    DEFAULT_CONFIG = {"user_choice_method": "random"}
+
+    DEFAULT_FOR_NEW_USERS = {
         "tenants": 1,
         "users_per_tenant": 1,
         "resource_management_workers":
             cfg.CONF.users_context.resource_management_workers,
-        "user_choice_method": "random",
     }
 
     def __init__(self, context):
-        self.credential = context["admin"]["credential"]
-        project_domain = (self.credential.project_domain_name or
-                          cfg.CONF.users_context.project_domain)
-        user_domain = (self.credential.user_domain_name or
-                       cfg.CONF.users_context.user_domain)
-        self.DEFAULT_CONFIG["project_domain"] = project_domain
-        self.DEFAULT_CONFIG["user_domain"] = user_domain
         super(UserGenerator, self).__init__(context)
 
+        deployment = objects.Deployment.get(context["task"]["deployment_uuid"])
+        existing_users = deployment.get_credentials_for("openstack")["users"]
+        if existing_users and not (set(self.config) - {"user_choice_method"}):
+            self.existing_users = existing_users
+        else:
+            self.existing_users = []
+            self.credential = context["admin"]["credential"]
+            project_domain = (self.credential.project_domain_name or
+                              cfg.CONF.users_context.project_domain)
+            user_domain = (self.credential.user_domain_name or
+                           cfg.CONF.users_context.user_domain)
+            self.DEFAULT_FOR_NEW_USERS["project_domain"] = project_domain
+            self.DEFAULT_FOR_NEW_USERS["user_domain"] = user_domain
+            with self.config.unlocked():
+                for key, value in self.DEFAULT_FOR_NEW_USERS.items():
+                    self.config.setdefault(key, value)
+
     def _remove_default_security_group(self):
         """Delete default security group for tenants."""
         clients = osclients.Clients(self.credential)
@@ -236,14 +257,8 @@ class UserGenerator(context.Context):
                    threads)
         self.context["users"] = []
 
-    @logging.log_task_wrapper(LOG.info, _("Enter context: `users`"))
+    def create_users(self):
-    def setup(self):
         """Create tenants and users, using the broker pattern."""
-        super(UserGenerator, self).setup()
-        self.context["users"] = []
-        self.context["tenants"] = {}
-        self.context["user_choice_method"] = self.config["user_choice_method"]
-
         threads = self.config["resource_management_workers"]
 
         LOG.debug("Creating %(tenants)d tenants using %(threads)s threads" %
@@ -267,9 +282,43 @@ class UserGenerator(context.Context):
                 ctx_name=self.get_name(),
                 msg=_("Failed to create the requested number of users."))
 
+    def use_existing_users(self):
+        LOG.debug("Using existing users")
+        for user_credential in self.existing_users:
+            user_clients = user_credential.clients()
+            user_id = user_clients.keystone.auth_ref.user_id
+            tenant_id = user_clients.keystone.auth_ref.project_id
+
+            if tenant_id not in self.context["tenants"]:
+                self.context["tenants"][tenant_id] = {
+                    "id": tenant_id,
+                    "name": user_credential.tenant_name
+                }
+
+            self.context["users"].append({
+                "credential": user_credential,
+                "id": user_id,
+                "tenant_id": tenant_id
+            })
+
+    @logging.log_task_wrapper(LOG.info, _("Enter context: `users`"))
+    def setup(self):
+        self.context["users"] = []
+        self.context["tenants"] = {}
+        self.context["user_choice_method"] = self.config["user_choice_method"]
+
+        if self.existing_users:
+            self.use_existing_users()
+        else:
+            self.create_users()
+
     @logging.log_task_wrapper(LOG.info, _("Exit context: `users`"))
     def cleanup(self):
         """Delete tenants and users, using the broker pattern."""
-        self._remove_default_security_group()
+        if self.existing_users:
-        self._delete_users()
+            # nothing to do here.
-        self._delete_tenants()
+            return
+        else:
+            self._remove_default_security_group()
+            self._delete_users()
+            self._delete_tenants()

tests/unit/plugins/openstack/context/keystone/test_users.py

@@ -24,7 +24,216 @@ from tests.unit import test
 CTX = "rally.plugins.openstack.context.keystone.users"
 
 
-class UserGeneratorTestCase(test.ScenarioTestCase):
+class UserGeneratorBaseTestCase(test.ScenarioTestCase):
+    def setUp(self):
+        super(UserGeneratorBaseTestCase, self).setUp()
+        self.osclients_patcher = mock.patch("%s.osclients" % CTX)
+        self.osclients = self.osclients_patcher.start()
+        self.addCleanup(self.osclients_patcher.stop)
+
+        self.deployment_patcher = mock.patch("%s.objects.Deployment.get" % CTX)
+        self.deployment_get = self.deployment_patcher.start()
+        self.addCleanup(self.deployment_patcher.stop)
+
+        self.deployment_uuid = "deployment_id"
+        self.admin_cred = mock.MagicMock()
+
+        self.context.update({
+            "config": {"users": {}},
+            "admin": {"credential": self.admin_cred},
+            "users": [],
+            "task": {"uuid": "task_id",
+                     "deployment_uuid": self.deployment_uuid}
+        })
+
+    def test___init__for_new_users(self):
+        deployment = self.deployment_get.return_value
+        deployment.get_credentials_for.return_value = {"users": []}
+        self.context["config"]["users"] = {
+            "tenants": 1, "users_per_tenant": 1,
+            "resource_management_workers": 1}
+
+        user_generator = users.UserGenerator(self.context)
+
+        self.assertEqual([], user_generator.existing_users)
+        self.assertEqual(self.admin_cred.project_domain_name,
+                         user_generator.config["project_domain"])
+        self.assertEqual(self.admin_cred.user_domain_name,
+                         user_generator.config["user_domain"])
+
+        self.deployment_get.assert_called_once_with(self.deployment_uuid)
+        deployment.get_credentials_for.assert_called_once_with("openstack")
+
+        self.deployment_get.reset_mock()
+        deployment.get_credentials_for.reset_mock()
+
+        # the case #2 - existing users are presented in deployment but
+        #   the user forces to create new ones
+        deployment.get_credentials_for.return_value = {"users": [mock.Mock()]}
+
+        user_generator = users.UserGenerator(self.context)
+
+        self.assertEqual([], user_generator.existing_users)
+        self.assertEqual(self.admin_cred.project_domain_name,
+                         user_generator.config["project_domain"])
+        self.assertEqual(self.admin_cred.user_domain_name,
+                         user_generator.config["user_domain"])
+
+        self.deployment_get.assert_called_once_with(self.deployment_uuid)
+        deployment.get_credentials_for.assert_called_once_with("openstack")
+
+    def test___init__for_existing_users(self):
+        deployment = self.deployment_get.return_value
+        foo_user = mock.Mock()
+        deployment.get_credentials_for.return_value = {"users": [foo_user]}
+
+        user_generator = users.UserGenerator(self.context)
+
+        self.assertEqual([foo_user], user_generator.existing_users)
+        self.assertEqual({"user_choice_method": "random"},
+                         user_generator.config)
+
+        self.deployment_get.assert_called_once_with(self.deployment_uuid)
+        deployment.get_credentials_for.assert_called_once_with("openstack")
+
+        self.deployment_get.reset_mock()
+        deployment.get_credentials_for.reset_mock()
+
+        # the case #2: the config with `user_choice_method` option
+        self.context["config"]["users"] = {"user_choice_method": "foo"}
+
+        user_generator = users.UserGenerator(self.context)
+
+        self.assertEqual([foo_user], user_generator.existing_users)
+        self.assertEqual({"user_choice_method": "foo"}, user_generator.config)
+
+        self.deployment_get.assert_called_once_with(self.deployment_uuid)
+        deployment.get_credentials_for.assert_called_once_with("openstack")
+
+    def test_setup(self):
+        user_generator = users.UserGenerator(self.context)
+        user_generator.use_existing_users = mock.Mock()
+        user_generator.create_users = mock.Mock()
+
+        # no existing users -> new users should be created
+        user_generator.existing_users = []
+
+        user_generator.setup()
+
+        user_generator.create_users.assert_called_once_with()
+        self.assertFalse(user_generator.use_existing_users.called)
+
+        user_generator.create_users.reset_mock()
+        user_generator.use_existing_users.reset_mock()
+
+        # existing_users is not empty -> existing users should be created
+        user_generator.existing_users = [mock.Mock()]
+
+        user_generator.setup()
+
+        user_generator.use_existing_users.assert_called_once_with()
+        self.assertFalse(user_generator.create_users.called)
+
+    def test_cleanup(self):
+        user_generator = users.UserGenerator(self.context)
+        user_generator._remove_default_security_group = mock.Mock()
+        user_generator._delete_users = mock.Mock()
+        user_generator._delete_tenants = mock.Mock()
+
+        # In case if existing users nothing should be done
+        user_generator.existing_users = [mock.Mock]
+
+        user_generator.cleanup()
+
+        self.assertFalse(user_generator._remove_default_security_group.called)
+        self.assertFalse(user_generator._delete_users.called)
+        self.assertFalse(user_generator._delete_tenants.called)
+
+        # In case when new users were created, the proper cleanup should be
+        #   performed
+        user_generator.existing_users = []
+
+        user_generator.cleanup()
+
+        user_generator._remove_default_security_group.assert_called_once_with()
+        user_generator._delete_users.assert_called_once_with()
+        user_generator._delete_tenants.assert_called_once_with()
+
+
+class UserGeneratorForExistingUsersTestCase(test.ScenarioTestCase):
+    def setUp(self):
+        super(UserGeneratorForExistingUsersTestCase, self).setUp()
+        self.osclients_patcher = mock.patch("%s.osclients" % CTX)
+        self.osclients = self.osclients_patcher.start()
+        self.addCleanup(self.osclients_patcher.stop)
+
+        self.deployment_patcher = mock.patch("%s.objects.Deployment.get" % CTX)
+        self.deployment_get = self.deployment_patcher.start()
+        self.addCleanup(self.deployment_patcher.stop)
+
+        self.deployment_uuid = "deployment_id"
+
+        self.context.update({
+            "config": {"users": {}},
+            "users": [],
+            "task": {"uuid": "task_id",
+                     "deployment_uuid": self.deployment_uuid}
+        })
+
+    def test_use_existing_users(self):
+        user1 = mock.MagicMock(tenant_id="1", user_id="1",
+                               tenant_name="proj", username="usr")
+        user2 = mock.MagicMock(tenant_id="1", user_id="2",
+                               tenant_name="proj", username="usr")
+        user3 = mock.MagicMock(tenant_id="2", user_id="3",
+                               tenant_name="proj", username="usr")
+
+        user_list = [user1, user2, user3]
+
+        class AuthRef(object):
+            USER_ID_COUNT = 0
+            PROJECT_ID_COUNT = 0
+
+            @property
+            def user_id(self):
+                self.USER_ID_COUNT += 1
+                return user_list[self.USER_ID_COUNT - 1].user_id
+
+            @property
+            def project_id(self):
+                self.PROJECT_ID_COUNT += 1
+                return user_list[self.PROJECT_ID_COUNT - 1].tenant_id
+
+        auth_ref = AuthRef()
+
+        user1.clients.return_value.keystone.auth_ref = auth_ref
+        user2.clients.return_value.keystone.auth_ref = auth_ref
+        user3.clients.return_value.keystone.auth_ref = auth_ref
+
+        deployment = self.deployment_get.return_value
+        deployment.get_credentials_for.return_value = {"users": user_list}
+
+        user_generator = users.UserGenerator(self.context)
+        user_generator.setup()
+
+        self.assertIn("users", self.context)
+        self.assertIn("tenants", self.context)
+        self.assertIn("user_choice_method", self.context)
+        self.assertEqual("random", self.context["user_choice_method"])
+        self.assertEqual(
+            [{"id": user1.user_id, "credential": user1,
+              "tenant_id": user1.tenant_id},
+             {"id": user2.user_id, "credential": user2,
+              "tenant_id": user2.tenant_id},
+             {"id": user3.user_id, "credential": user3,
+              "tenant_id": user3.tenant_id}], self.context["users"]
+        )
+        self.assertEqual({"1": {"id": "1", "name": user1.tenant_name},
+                          "2": {"id": "2", "name": user3.tenant_name}},
+                         self.context["tenants"])
+
+
+class UserGeneratorForNewUsersTestCase(test.ScenarioTestCase):
 
     tenants_num = 1
     users_per_tenant = 5
@@ -32,9 +241,19 @@ class UserGeneratorTestCase(test.ScenarioTestCase):
     threads = 10
 
     def setUp(self):
-        super(UserGeneratorTestCase, self).setUp()
+        super(UserGeneratorForNewUsersTestCase, self).setUp()
         self.osclients_patcher = mock.patch("%s.osclients" % CTX)
         self.osclients = self.osclients_patcher.start()
+        self.addCleanup(self.osclients_patcher.stop)
+
+        self.deployment_patcher = mock.patch("%s.objects.Deployment.get" % CTX)
+        self.deployment_get = self.deployment_patcher.start()
+        self.addCleanup(self.deployment_patcher.stop)
+
+        # Force the case of creating new users
+        deployment = self.deployment_get.return_value
+        deployment.get_credentials_for.return_value = {"users": []}
+
         self.context.update({
             "config": {
                 "users": {
@@ -45,13 +264,9 @@ class UserGeneratorTestCase(test.ScenarioTestCase):
             },
             "admin": {"credential": mock.MagicMock()},
             "users": [],
-            "task": {"uuid": "task_id"}
+            "task": {"uuid": "task_id", "deployment_uuid": "dep_uuid"}
         })
 
-    def tearDown(self):
-        self.osclients_patcher.stop()
-        super(UserGeneratorTestCase, self).tearDown()
-
     @mock.patch("%s.network.wrap" % CTX)
     def test__remove_default_security_group_not_needed(self, mock_wrap):
         services = {"compute": consts.Service.NOVA}
@@ -203,6 +418,7 @@ class UserGeneratorTestCase(test.ScenarioTestCase):
                              self.users_num)
             self.assertEqual(len(ctx.context["tenants"]),
                              self.tenants_num)
+
             self.assertEqual("random", ctx.context["user_choice_method"])
 
         # Cleanup (called by content manager)
@@ -289,7 +505,7 @@ class UserGeneratorTestCase(test.ScenarioTestCase):
                 }
             },
             "admin": {"credential": credential},
-            "task": {"uuid": "task_id"}
+            "task": {"uuid": "task_id", "deployment_uuid": "deployment_id"}
         }
 
         user_generator = users.UserGenerator(config)
@@ -311,7 +527,7 @@ class UserGeneratorTestCase(test.ScenarioTestCase):
                 }
             },
             "admin": {"credential": credential},
-            "task": {"uuid": "task_id"}
+            "task": {"uuid": "task_id", "deployment_uuid": "deployment_id"}
         }
 
         user_generator = users.UserGenerator(config)

tests/unit/rally_jobs/test_jobs.py

@@ -73,7 +73,7 @@ class RallyJobsTestCase(test.TestCase):
                         if not isinstance(args, dict):
                             raise TypeError(
                                 "args file %s must be dict in yaml or json "
-                                "presenatation" % args_file)
+                                "presentation" % args_file)
 
                     task_inst = api._Task(api.API(skip_db_check=True))
                     task = task_inst.render_template(