Merge "[existing users] Restore original quotas"

rally/plugins/openstack/context/quotas/cinder_quotas.py

@@ -43,3 +43,8 @@ class CinderQuotas(object):
 
     def delete(self, tenant_id):
         self.clients.cinder().quotas.delete(tenant_id)
+
+    def get(self, tenant_id):
+        response = self.clients.cinder().quotas.get(tenant_id)
+        return dict([(k, getattr(response, k))
+                     for k in self.QUOTAS_SCHEMA["properties"]])

rally/plugins/openstack/context/quotas/designate_quotas.py

@@ -47,3 +47,10 @@ class DesignateQuotas(object):
 
     def delete(self, tenant_id):
         self.clients.designate().quotas.reset(tenant_id)
+
+    def get(self, tenant_id):
+        # NOTE(andreykurilin): we have broken designate jobs, so I can't check
+        #   that this method is right :(
+        response = self.clients.designate().quotas.get(tenant_id)
+        return dict([(k, response.get(k))
+                     for k in self.QUOTAS_SCHEMA["properties"]])

rally/plugins/openstack/context/quotas/manila_quotas.py

@@ -52,3 +52,8 @@ class ManilaQuotas(object):
 
     def delete(self, tenant_id):
         self.clients.manila().quotas.delete(tenant_id)
+
+    def get(self, tenant_id):
+        response = self.clients.manila().quotas.get(tenant_id)
+        return dict([(k, getattr(response, k))
+                     for k in self.QUOTAS_SCHEMA["properties"]])

rally/plugins/openstack/context/quotas/neutron_quotas.py

@@ -73,3 +73,6 @@ class NeutronQuotas(object):
     def delete(self, tenant_id):
         # Reset quotas to defaults and tag database objects as deleted
         self.clients.neutron().delete_quota(tenant_id)
+
+    def get(self, tenant_id):
+        return self.clients.neutron().show_quota(tenant_id)["quota"]

rally/plugins/openstack/context/quotas/nova_quotas.py

@@ -88,3 +88,8 @@ class NovaQuotas(object):
     def delete(self, tenant_id):
         # Reset quotas to defaults and tag database objects as deleted
         self.clients.nova().quotas.delete(tenant_id)
+
+    def get(self, tenant_id):
+        response = self.clients.nova().quotas.get(tenant_id)
+        return dict([(k, getattr(response, k))
+                     for k in self.QUOTAS_SCHEMA["properties"]])

rally/plugins/openstack/context/quotas/quotas.py

@@ -58,6 +58,7 @@ class Quotas(context.Context):
             "designate": designate_quotas.DesignateQuotas(self.clients),
             "neutron": neutron_quotas.NeutronQuotas(self.clients)
         }
+        self.original_quotas = []
 
     def _service_has_quotas(self, service):
         return len(self.config.get(service, {})) > 0
@@ -67,11 +68,27 @@ class Quotas(context.Context):
         for tenant_id in self.context["tenants"]:
             for service in self.manager:
                 if self._service_has_quotas(service):
+                    # NOTE(andreykurilin): in case of existing users it is
+                    #   required to restore original quotas instead of reset
+                    #   to default ones.
+                    if "existing_users" in self.context:
+                        self.original_quotas.append(
+                            (service, tenant_id,
+                             self.manager[service].get(tenant_id)))
                     self.manager[service].update(tenant_id,
                                                  **self.config[service])
 
-    @logging.log_task_wrapper(LOG.info, _("Exit context: `quotas`"))
+    def _restore_quotas(self):
-    def cleanup(self):
+        for service, tenant_id, quotas in self.original_quotas:
+            try:
+                self.manager[service].update(tenant_id, **quotas)
+            except Exception as e:
+                LOG.warning("Failed to restore quotas for tenant %(tenant_id)s"
+                            " in service %(service)s \n reason: %(exc)s" %
+                            {"tenant_id": tenant_id, "service": service,
+                             "exc": e})
+
+    def _delete_quotas(self):
         for service in self.manager:
             if self._service_has_quotas(service):
                 for tenant_id in self.context["tenants"]:
@@ -83,3 +100,11 @@ class Quotas(context.Context):
                                     "\n reason: %(exc)s"
                                     % {"tenant_id": tenant_id,
                                        "service": service, "exc": e})
+
+    @logging.log_task_wrapper(LOG.info, _("Exit context: `quotas`"))
+    def cleanup(self):
+        if self.original_quotas:
+            # existing users
+            self._restore_quotas()
+        else:
+            self._delete_quotas()

tests/unit/plugins/openstack/context/quotas/test_cinder_quotas.py

@@ -39,3 +39,14 @@ class CinderQuotasTestCase(test.TestCase):
         tenant_id = mock.MagicMock()
         cinder_quo.delete(tenant_id)
         mock_clients.cinder().quotas.delete.assert_called_once_with(tenant_id)
+
+    def test_get(self):
+        tenant_id = "tenant_id"
+        quotas = {"gigabytes": "gb", "snapshots": "ss", "volumes": "v"}
+        quota_set = mock.MagicMock(**quotas)
+        clients = mock.MagicMock()
+        clients.cinder.return_value.quotas.get.return_value = quota_set
+        cinder_quo = cinder_quotas.CinderQuotas(clients)
+
+        self.assertEqual(quotas, cinder_quo.get(tenant_id))
+        clients.cinder().quotas.get.assert_called_once_with(tenant_id)

tests/unit/plugins/openstack/context/quotas/test_designate_quotas.py

@@ -20,10 +20,9 @@ from tests.unit import test
 
 class DesignateQuotasTestCase(test.TestCase):
 
-    @mock.patch("rally.plugins.openstack.context."
+    def test_update(self):
-                "quotas.quotas.osclients.Clients")
+        clients = mock.MagicMock()
-    def test_update(self, mock_clients):
+        quotas = designate_quotas.DesignateQuotas(clients)
-        quotas = designate_quotas.DesignateQuotas(mock_clients)
         tenant_id = mock.MagicMock()
         quotas_values = {
             "domains": 5,
@@ -32,14 +31,23 @@ class DesignateQuotasTestCase(test.TestCase):
             "recordset_records": 20,
         }
         quotas.update(tenant_id, **quotas_values)
-        mock_clients.designate().quotas.update.assert_called_once_with(
+        clients.designate().quotas.update.assert_called_once_with(
             tenant_id, quotas_values)
 
-    @mock.patch("rally.plugins.openstack.context."
+    def test_delete(self):
-                "quotas.quotas.osclients.Clients")
+        clients = mock.MagicMock()
-    def test_delete(self, mock_clients):
+        quotas = designate_quotas.DesignateQuotas(clients)
-        quotas = designate_quotas.DesignateQuotas(mock_clients)
         tenant_id = mock.MagicMock()
         quotas.delete(tenant_id)
-        mock_clients.designate().quotas.reset.assert_called_once_with(
+        clients.designate().quotas.reset.assert_called_once_with(tenant_id)
-            tenant_id)
+
+    def test_get(self):
+        tenant_id = "tenant_id"
+        quotas = {"domains": -1, "domain_recordsets": 2, "domain_records": 3,
+                  "recordset_records": 3}
+        clients = mock.MagicMock()
+        clients.designate.return_value.quotas.get.return_value = quotas
+        designate_quo = designate_quotas.DesignateQuotas(clients)
+
+        self.assertEqual(quotas, designate_quo.get(tenant_id))
+        clients.designate().quotas.get.assert_called_once_with(tenant_id)

tests/unit/plugins/openstack/context/quotas/test_manila_quotas.py

@@ -18,15 +18,12 @@ import mock
 from rally.plugins.openstack.context.quotas import manila_quotas
 from tests.unit import test
 
-CLIENTS_CLASS = (
-    "rally.plugins.openstack.context.quotas.quotas.osclients.Clients")
-
 
 class ManilaQuotasTestCase(test.TestCase):
 
-    @mock.patch(CLIENTS_CLASS)
+    def test_update(self):
-    def test_update(self, mock_clients):
+        clients = mock.MagicMock()
-        instance = manila_quotas.ManilaQuotas(mock_clients)
+        instance = manila_quotas.ManilaQuotas(clients)
         tenant_id = mock.MagicMock()
         quotas_values = {
             "shares": 10,
@@ -38,15 +35,27 @@ class ManilaQuotasTestCase(test.TestCase):
 
         instance.update(tenant_id, **quotas_values)
 
-        mock_clients.manila.return_value.quotas.update.assert_called_once_with(
+        clients.manila.return_value.quotas.update.assert_called_once_with(
             tenant_id, **quotas_values)
 
-    @mock.patch(CLIENTS_CLASS)
+    def test_delete(self):
-    def test_delete(self, mock_clients):
+        clients = mock.MagicMock()
-        instance = manila_quotas.ManilaQuotas(mock_clients)
+        instance = manila_quotas.ManilaQuotas(clients)
         tenant_id = mock.MagicMock()
 
         instance.delete(tenant_id)
 
-        mock_clients.manila.return_value.quotas.delete.assert_called_once_with(
+        clients.manila.return_value.quotas.delete.assert_called_once_with(
             tenant_id)
+
+    def test_get(self):
+        tenant_id = "tenant_id"
+        quotas = {"gigabytes": "gb", "snapshots": "ss", "shares": "v",
+                  "snapshot_gigabytes": "sg", "share_networks": "sn"}
+        quota_set = mock.MagicMock(**quotas)
+        clients = mock.MagicMock()
+        clients.manila.return_value.quotas.get.return_value = quota_set
+        manila_quo = manila_quotas.ManilaQuotas(clients)
+
+        self.assertEqual(quotas, manila_quo.get(tenant_id))
+        clients.manila().quotas.get.assert_called_once_with(tenant_id)

tests/unit/plugins/openstack/context/quotas/test_neutron_quotas.py

@@ -14,18 +14,14 @@
 
 import mock
 
-from rally.plugins.openstack.context.quotas import neutron_quotas as quotas
+from rally.plugins.openstack.context.quotas import neutron_quotas
 from tests.unit import test
 
 
 class NeutronQuotasTestCase(test.TestCase):
-
+    def setUp(self):
-    @mock.patch("rally.plugins.openstack.context."
+        super(NeutronQuotasTestCase, self).setUp()
-                "quotas.quotas.osclients.Clients")
+        self.quotas = {
-    def test_update(self, mock_clients):
-        neutron_quotas = quotas.NeutronQuotas(mock_clients)
-        tenant_id = mock.MagicMock()
-        quotas_values = {
             "network": 20,
             "subnet": 20,
             "port": 100,
@@ -34,15 +30,29 @@ class NeutronQuotasTestCase(test.TestCase):
             "security_group": 100,
             "security_group_rule": 100
         }
-        neutron_quotas.update(tenant_id, **quotas_values)
-        body = {"quota": quotas_values}
-        mock_clients.neutron().update_quota.assert_called_once_with(tenant_id,
-                                                                    body=body)
 
-    @mock.patch("rally.plugins.openstack.context."
+    def test_update(self):
-                "quotas.quotas.osclients.Clients")
+        clients = mock.MagicMock()
-    def test_delete(self, mock_clients):
+        neutron_quo = neutron_quotas.NeutronQuotas(clients)
-        neutron_quotas = quotas.NeutronQuotas(mock_clients)
         tenant_id = mock.MagicMock()
-        neutron_quotas.delete(tenant_id)
+        neutron_quo.update(tenant_id, **self.quotas)
-        mock_clients.neutron().delete_quota.assert_called_once_with(tenant_id)
+        body = {"quota": self.quotas}
+        clients.neutron().update_quota.assert_called_once_with(tenant_id,
+                                                               body=body)
+
+    def test_delete(self):
+        clients = mock.MagicMock()
+        neutron_quo = neutron_quotas.NeutronQuotas(clients)
+        tenant_id = mock.MagicMock()
+        neutron_quo.delete(tenant_id)
+        clients.neutron().delete_quota.assert_called_once_with(tenant_id)
+
+    def test_get(self):
+        tenant_id = "tenant_id"
+        clients = mock.MagicMock()
+        clients.neutron.return_value.show_quota.return_value = {
+            "quota": self.quotas}
+        neutron_quo = neutron_quotas.NeutronQuotas(clients)
+
+        self.assertEqual(self.quotas, neutron_quo.get(tenant_id))
+        clients.neutron().show_quota.assert_called_once_with(tenant_id)

tests/unit/plugins/openstack/context/quotas/test_nova_quotas.py

@@ -14,18 +14,15 @@
 
 import mock
 
-from rally.plugins.openstack.context.quotas import nova_quotas as quotas
+from rally.plugins.openstack.context.quotas import nova_quotas
 from tests.unit import test
 
 
 class NovaQuotasTestCase(test.TestCase):
 
-    @mock.patch("rally.plugins.openstack.context."
+    def setUp(self):
-                "quotas.quotas.osclients.Clients")
+        super(NovaQuotasTestCase, self).setUp()
-    def test_update(self, mock_clients):
+        self.quotas = {
-        nova_quotas = quotas.NovaQuotas(mock_clients)
-        tenant_id = mock.MagicMock()
-        quotas_values = {
             "instances": 10,
             "cores": 100,
             "ram": 100000,
@@ -37,16 +34,32 @@ class NovaQuotasTestCase(test.TestCase):
             "injected_file_path_bytes": 1024,
             "key_pairs": 50,
             "security_groups": 50,
-            "security_group_rules": 50
+            "security_group_rules": 50,
+            "server_group_members": 777,
+            "server_groups": 33
         }
-        nova_quotas.update(tenant_id, **quotas_values)
-        mock_clients.nova().quotas.update.assert_called_once_with(
-            tenant_id, **quotas_values)
 
-    @mock.patch("rally.plugins.openstack.context."
+    def test_update(self):
-                "quotas.quotas.osclients.Clients")
+        clients = mock.MagicMock()
-    def test_delete(self, mock_clients):
+        nova_quo = nova_quotas.NovaQuotas(clients)
-        nova_quotas = quotas.NovaQuotas(mock_clients)
         tenant_id = mock.MagicMock()
-        nova_quotas.delete(tenant_id)
+        nova_quo.update(tenant_id, **self.quotas)
-        mock_clients.nova().quotas.delete.assert_called_once_with(tenant_id)
+        clients.nova().quotas.update.assert_called_once_with(tenant_id,
+                                                             **self.quotas)
+
+    def test_delete(self):
+        clients = mock.MagicMock()
+        nova_quo = nova_quotas.NovaQuotas(clients)
+        tenant_id = mock.MagicMock()
+        nova_quo.delete(tenant_id)
+        clients.nova().quotas.delete.assert_called_once_with(tenant_id)
+
+    def test_get(self):
+        tenant_id = "tenant_id"
+        quota_set = mock.MagicMock(**self.quotas)
+        clients = mock.MagicMock()
+        clients.nova.return_value.quotas.get.return_value = quota_set
+        nova_quo = nova_quotas.NovaQuotas(clients)
+
+        self.assertEqual(self.quotas, nova_quo.get(tenant_id))
+        clients.nova().quotas.get.assert_called_once_with(tenant_id)

tests/unit/plugins/openstack/context/quotas/test_quotas.py

@@ -20,10 +20,11 @@ import ddt
 import jsonschema
 import mock
 
+from rally.common import logging
 from rally.plugins.openstack.context.quotas import quotas
 from tests.unit import test
 
-QUOTAS_PATH = "rally.plugins.openstack.context.quotas."
+QUOTAS_PATH = "rally.plugins.openstack.context.quotas"
 
 
 @ddt.ddt
@@ -135,12 +136,14 @@ class QuotasTestCase(test.TestCase):
             except jsonschema.ValidationError:
                 self.fail("Valid quota keys are optional")
 
-    @mock.patch("rally.plugins.openstack.context."
+    @mock.patch("%s.quotas.osclients.Clients" % QUOTAS_PATH)
-                "quotas.quotas.osclients.Clients")
+    @mock.patch("%s.cinder_quotas.CinderQuotas" % QUOTAS_PATH)
-    @mock.patch("rally.plugins.openstack.context."
+    @ddt.data(True, False)
-                "quotas.cinder_quotas.CinderQuotas")
+    def test_cinder_quotas(self, ex_users, mock_cinder_quotas, mock_clients):
-    def test_cinder_quotas(self, mock_cinder_quotas, mock_clients):
+        cinder_quo = mock_cinder_quotas.return_value
         ctx = copy.deepcopy(self.context)
+        if ex_users:
+            ctx["existing_users"] = None
         ctx["config"]["quotas"] = {
             "cinder": {
                 "volumes": self.unlimited,
@@ -151,29 +154,33 @@ class QuotasTestCase(test.TestCase):
 
         tenants = ctx["tenants"]
         cinder_quotas = ctx["config"]["quotas"]["cinder"]
+        cinder_quo.get.return_value = cinder_quotas
         with quotas.Quotas(ctx) as quotas_ctx:
             quotas_ctx.setup()
-            expected_setup_calls = []
+            if ex_users:
-            for tenant in tenants:
+                self.assertEqual([mock.call(tenant) for tenant in tenants],
-                expected_setup_calls.append(mock.call()
+                                 cinder_quo.get.call_args_list)
-                                                .update(tenant,
+            self.assertEqual([mock.call(tenant, **cinder_quotas)
-                                                        **cinder_quotas))
+                              for tenant in tenants],
-            mock_cinder_quotas.assert_has_calls(
+                             cinder_quo.update.call_args_list)
-                expected_setup_calls, any_order=True)
             mock_cinder_quotas.reset_mock()
 
-        expected_cleanup_calls = []
+        if ex_users:
-        for tenant in tenants:
+            self.assertEqual([mock.call(tenant, **cinder_quotas)
-            expected_cleanup_calls.append(mock.call().delete(tenant))
+                              for tenant in tenants],
-        mock_cinder_quotas.assert_has_calls(
+                             cinder_quo.update.call_args_list)
-            expected_cleanup_calls, any_order=True)
+        else:
+            self.assertEqual([mock.call(tenant) for tenant in tenants],
+                             cinder_quo.delete.call_args_list)
 
-    @mock.patch("rally.plugins.openstack.context."
+    @mock.patch("%s.quotas.osclients.Clients" % QUOTAS_PATH)
-                "quotas.quotas.osclients.Clients")
+    @mock.patch("%s.nova_quotas.NovaQuotas" % QUOTAS_PATH)
-    @mock.patch("rally.plugins.openstack.context."
+    @ddt.data(True, False)
-                "quotas.nova_quotas.NovaQuotas")
+    def test_nova_quotas(self, ex_users, mock_nova_quotas, mock_clients):
-    def test_nova_quotas(self, mock_nova_quotas, mock_clients):
+        nova_quo = mock_nova_quotas.return_value
         ctx = copy.deepcopy(self.context)
+        if ex_users:
+            ctx["existing_users"] = None
 
         ctx["config"]["quotas"] = {
             "nova": {
@@ -192,30 +199,35 @@ class QuotasTestCase(test.TestCase):
             }
         }
 
+        tenants = ctx["tenants"]
         nova_quotas = ctx["config"]["quotas"]["nova"]
+        nova_quo.get.return_value = nova_quotas
         with quotas.Quotas(ctx) as quotas_ctx:
             quotas_ctx.setup()
-            expected_setup_calls = []
+            if ex_users:
-            for tenant in ctx["tenants"]:
+                self.assertEqual([mock.call(tenant) for tenant in tenants],
-                expected_setup_calls.append(mock.call()
+                                 nova_quo.get.call_args_list)
-                                                .update(tenant,
+            self.assertEqual([mock.call(tenant, **nova_quotas)
-                                                        **nova_quotas))
+                              for tenant in tenants],
-            mock_nova_quotas.assert_has_calls(
+                             nova_quo.update.call_args_list)
-                expected_setup_calls, any_order=True)
             mock_nova_quotas.reset_mock()
 
-        expected_cleanup_calls = []
+        if ex_users:
-        for tenant in ctx["tenants"]:
+            self.assertEqual([mock.call(tenant, **nova_quotas)
-            expected_cleanup_calls.append(mock.call().delete(tenant))
+                              for tenant in tenants],
-        mock_nova_quotas.assert_has_calls(
+                             nova_quo.update.call_args_list)
-            expected_cleanup_calls, any_order=True)
+        else:
+            self.assertEqual([mock.call(tenant) for tenant in tenants],
+                             nova_quo.delete.call_args_list)
 
-    @mock.patch("rally.plugins.openstack.context."
+    @mock.patch("%s.quotas.osclients.Clients" % QUOTAS_PATH)
-                "quotas.quotas.osclients.Clients")
+    @mock.patch("%s.neutron_quotas.NeutronQuotas" % QUOTAS_PATH)
-    @mock.patch("rally.plugins.openstack.context."
+    @ddt.data(True, False)
-                "quotas.neutron_quotas.NeutronQuotas")
+    def test_neutron_quotas(self, ex_users, mock_neutron_quotas, mock_clients):
-    def test_neutron_quotas(self, mock_neutron_quotas, mock_clients):
+        neutron_quo = mock_neutron_quotas.return_value
         ctx = copy.deepcopy(self.context)
+        if ex_users:
+            ctx["existing_users"] = None
 
         ctx["config"]["quotas"] = {
             "neutron": {
@@ -229,23 +241,26 @@ class QuotasTestCase(test.TestCase):
             }
         }
 
+        tenants = ctx["tenants"]
         neutron_quotas = ctx["config"]["quotas"]["neutron"]
+        neutron_quo.get.return_value = neutron_quotas
         with quotas.Quotas(ctx) as quotas_ctx:
             quotas_ctx.setup()
-            expected_setup_calls = []
+            if ex_users:
-            for tenant in ctx["tenants"]:
+                self.assertEqual([mock.call(tenant) for tenant in tenants],
-                expected_setup_calls.append(mock.call()
+                                 neutron_quo.get.call_args_list)
-                                                .update(tenant,
+            self.assertEqual([mock.call(tenant, **neutron_quotas)
-                                                        **neutron_quotas))
+                              for tenant in tenants],
-            mock_neutron_quotas.assert_has_calls(
+                             neutron_quo.update.call_args_list)
-                expected_setup_calls, any_order=True)
+            neutron_quo.reset_mock()
-            mock_neutron_quotas.reset_mock()
 
-        expected_cleanup_calls = []
+        if ex_users:
-        for tenant in ctx["tenants"]:
+            self.assertEqual([mock.call(tenant, **neutron_quotas)
-            expected_cleanup_calls.append(mock.call().delete(tenant))
+                              for tenant in tenants],
-        mock_neutron_quotas.assert_has_calls(
+                             neutron_quo.update.call_args_list)
-            expected_cleanup_calls, any_order=True)
+        else:
+            self.assertEqual([mock.call(tenant) for tenant in tenants],
+                             neutron_quo.delete.call_args_list)
 
     @mock.patch("rally.plugins.openstack.context."
                 "quotas.quotas.osclients.Clients")
@@ -285,15 +300,24 @@ class QuotasTestCase(test.TestCase):
     )
     @ddt.unpack
     def test_exception_during_cleanup(self, quotas_ctxt, quotas_class_path):
-        with mock.patch(QUOTAS_PATH + quotas_class_path) as mock_quotas:
+        quotas_path = "%s.%s" % (QUOTAS_PATH, quotas_class_path)
-            mock_quotas.delete.side_effect = type(
+        with mock.patch(quotas_path) as mock_quotas:
-                "ExceptionDuringCleanup", (Exception, ), {})
+            mock_quotas.return_value.update.side_effect = Exception
 
             ctx = copy.deepcopy(self.context)
             ctx["config"]["quotas"] = quotas_ctxt
 
+            quotas_instance = quotas.Quotas(ctx)
+            quotas_instance.original_quotas = []
+            for service in quotas_ctxt:
+                for tenant in self.context["tenants"]:
+                    quotas_instance.original_quotas.append(
+                        (service, tenant, quotas_ctxt[service]))
             # NOTE(boris-42): ensure that cleanup didn't raise exceptions.
-            quotas.Quotas(ctx).cleanup()
+            with logging.LogCatcher(quotas.LOG) as log:
+                quotas_instance.cleanup()
 
-            self.assertEqual(mock_quotas.return_value.delete.call_count,
+                log.assertInLogs("Failed to restore quotas for tenant")
+
+            self.assertEqual(mock_quotas.return_value.update.call_count,
                              len(self.context["tenants"]))