[existing users] Restore original quotas

Behaviors of quotas cleanup should be different in case of existing users and
new ones(created by users context).

In case of existing users, we should not delete quotas, since we should not
change user pre-defined data.
In case of new users, we can't do the same as with existing users. Restore
default quotas means that services databases will continue storing quotas even
after tenant removement. So we need to remove quotas in this case.

Closes-Bug: #1595578

Change-Id: I6d42942f7fcdc67f221dc737495663eec065c8e4

rally/plugins/openstack/context/quotas/cinder_quotas.py

@@ -43,3 +43,8 @@ class CinderQuotas(object):
 
     def delete(self, tenant_id):
         self.clients.cinder().quotas.delete(tenant_id)
+
+    def get(self, tenant_id):
+        response = self.clients.cinder().quotas.get(tenant_id)
+        return dict([(k, getattr(response, k))
+                     for k in self.QUOTAS_SCHEMA["properties"]])

rally/plugins/openstack/context/quotas/designate_quotas.py

@@ -47,3 +47,10 @@ class DesignateQuotas(object):
 
     def delete(self, tenant_id):
         self.clients.designate().quotas.reset(tenant_id)
+
+    def get(self, tenant_id):
+        # NOTE(andreykurilin): we have broken designate jobs, so I can't check
+        #   that this method is right :(
+        response = self.clients.designate().quotas.get(tenant_id)
+        return dict([(k, response.get(k))
+                     for k in self.QUOTAS_SCHEMA["properties"]])

rally/plugins/openstack/context/quotas/manila_quotas.py

@@ -52,3 +52,8 @@ class ManilaQuotas(object):
 
     def delete(self, tenant_id):
         self.clients.manila().quotas.delete(tenant_id)
+
+    def get(self, tenant_id):
+        response = self.clients.manila().quotas.get(tenant_id)
+        return dict([(k, getattr(response, k))
+                     for k in self.QUOTAS_SCHEMA["properties"]])

rally/plugins/openstack/context/quotas/neutron_quotas.py

@@ -73,3 +73,6 @@ class NeutronQuotas(object):
     def delete(self, tenant_id):
         # Reset quotas to defaults and tag database objects as deleted
         self.clients.neutron().delete_quota(tenant_id)
+
+    def get(self, tenant_id):
+        return self.clients.neutron().show_quota(tenant_id)["quota"]

rally/plugins/openstack/context/quotas/nova_quotas.py

@@ -88,3 +88,8 @@ class NovaQuotas(object):
     def delete(self, tenant_id):
         # Reset quotas to defaults and tag database objects as deleted
         self.clients.nova().quotas.delete(tenant_id)
+
+    def get(self, tenant_id):
+        response = self.clients.nova().quotas.get(tenant_id)
+        return dict([(k, getattr(response, k))
+                     for k in self.QUOTAS_SCHEMA["properties"]])

rally/plugins/openstack/context/quotas/quotas.py

@@ -58,6 +58,7 @@ class Quotas(context.Context):
             "designate": designate_quotas.DesignateQuotas(self.clients),
             "neutron": neutron_quotas.NeutronQuotas(self.clients)
         }
+        self.original_quotas = []
 
     def _service_has_quotas(self, service):
         return len(self.config.get(service, {})) > 0
@@ -67,11 +68,27 @@ class Quotas(context.Context):
         for tenant_id in self.context["tenants"]:
             for service in self.manager:
                 if self._service_has_quotas(service):
+                    # NOTE(andreykurilin): in case of existing users it is
+                    #   required to restore original quotas instead of reset
+                    #   to default ones.
+                    if "existing_users" in self.context:
+                        self.original_quotas.append(
+                            (service, tenant_id,
+                             self.manager[service].get(tenant_id)))
                     self.manager[service].update(tenant_id,
                                                  **self.config[service])
 
-    @logging.log_task_wrapper(LOG.info, _("Exit context: `quotas`"))
+    def _restore_quotas(self):
-    def cleanup(self):
+        for service, tenant_id, quotas in self.original_quotas:
+            try:
+                self.manager[service].update(tenant_id, **quotas)
+            except Exception as e:
+                LOG.warning("Failed to restore quotas for tenant %(tenant_id)s"
+                            " in service %(service)s \n reason: %(exc)s" %
+                            {"tenant_id": tenant_id, "service": service,
+                             "exc": e})
+
+    def _delete_quotas(self):
         for service in self.manager:
             if self._service_has_quotas(service):
                 for tenant_id in self.context["tenants"]:
@@ -83,3 +100,11 @@ class Quotas(context.Context):
                                     "\n reason: %(exc)s"
                                     % {"tenant_id": tenant_id,
                                        "service": service, "exc": e})
+
+    @logging.log_task_wrapper(LOG.info, _("Exit context: `quotas`"))
+    def cleanup(self):
+        if self.original_quotas:
+            # existing users
+            self._restore_quotas()
+        else:
+            self._delete_quotas()

tests/unit/plugins/openstack/context/quotas/test_cinder_quotas.py

@@ -39,3 +39,14 @@ class CinderQuotasTestCase(test.TestCase):
         tenant_id = mock.MagicMock()
         cinder_quo.delete(tenant_id)
         mock_clients.cinder().quotas.delete.assert_called_once_with(tenant_id)
+
+    def test_get(self):
+        tenant_id = "tenant_id"
+        quotas = {"gigabytes": "gb", "snapshots": "ss", "volumes": "v"}
+        quota_set = mock.MagicMock(**quotas)
+        clients = mock.MagicMock()
+        clients.cinder.return_value.quotas.get.return_value = quota_set
+        cinder_quo = cinder_quotas.CinderQuotas(clients)
+
+        self.assertEqual(quotas, cinder_quo.get(tenant_id))
+        clients.cinder().quotas.get.assert_called_once_with(tenant_id)

tests/unit/plugins/openstack/context/quotas/test_designate_quotas.py

@@ -20,10 +20,9 @@ from tests.unit import test
 
 class DesignateQuotasTestCase(test.TestCase):
 
-    @mock.patch("rally.plugins.openstack.context."
+    def test_update(self):
-                "quotas.quotas.osclients.Clients")
+        clients = mock.MagicMock()
-    def test_update(self, mock_clients):
+        quotas = designate_quotas.DesignateQuotas(clients)
-        quotas = designate_quotas.DesignateQuotas(mock_clients)
         tenant_id = mock.MagicMock()
         quotas_values = {
             "domains": 5,
@@ -32,14 +31,23 @@ class DesignateQuotasTestCase(test.TestCase):
             "recordset_records": 20,
         }
         quotas.update(tenant_id, **quotas_values)
-        mock_clients.designate().quotas.update.assert_called_once_with(
+        clients.designate().quotas.update.assert_called_once_with(
             tenant_id, quotas_values)
 
-    @mock.patch("rally.plugins.openstack.context."
+    def test_delete(self):
-                "quotas.quotas.osclients.Clients")
+        clients = mock.MagicMock()
-    def test_delete(self, mock_clients):
+        quotas = designate_quotas.DesignateQuotas(clients)
-        quotas = designate_quotas.DesignateQuotas(mock_clients)
         tenant_id = mock.MagicMock()
         quotas.delete(tenant_id)
-        mock_clients.designate().quotas.reset.assert_called_once_with(
+        clients.designate().quotas.reset.assert_called_once_with(tenant_id)
-            tenant_id)
+
+    def test_get(self):
+        tenant_id = "tenant_id"
+        quotas = {"domains": -1, "domain_recordsets": 2, "domain_records": 3,
+                  "recordset_records": 3}
+        clients = mock.MagicMock()
+        clients.designate.return_value.quotas.get.return_value = quotas
+        designate_quo = designate_quotas.DesignateQuotas(clients)
+
+        self.assertEqual(quotas, designate_quo.get(tenant_id))
+        clients.designate().quotas.get.assert_called_once_with(tenant_id)

tests/unit/plugins/openstack/context/quotas/test_manila_quotas.py

@@ -18,15 +18,12 @@ import mock
 from rally.plugins.openstack.context.quotas import manila_quotas
 from tests.unit import test
 
-CLIENTS_CLASS = (
-    "rally.plugins.openstack.context.quotas.quotas.osclients.Clients")
-
 
 class ManilaQuotasTestCase(test.TestCase):
 
-    @mock.patch(CLIENTS_CLASS)
+    def test_update(self):
-    def test_update(self, mock_clients):
+        clients = mock.MagicMock()
-        instance = manila_quotas.ManilaQuotas(mock_clients)
+        instance = manila_quotas.ManilaQuotas(clients)
         tenant_id = mock.MagicMock()
         quotas_values = {
             "shares": 10,
@@ -38,15 +35,27 @@ class ManilaQuotasTestCase(test.TestCase):
 
         instance.update(tenant_id, **quotas_values)
 
-        mock_clients.manila.return_value.quotas.update.assert_called_once_with(
+        clients.manila.return_value.quotas.update.assert_called_once_with(
             tenant_id, **quotas_values)
 
-    @mock.patch(CLIENTS_CLASS)
+    def test_delete(self):
-    def test_delete(self, mock_clients):
+        clients = mock.MagicMock()
-        instance = manila_quotas.ManilaQuotas(mock_clients)
+        instance = manila_quotas.ManilaQuotas(clients)
         tenant_id = mock.MagicMock()
 
         instance.delete(tenant_id)
 
-        mock_clients.manila.return_value.quotas.delete.assert_called_once_with(
+        clients.manila.return_value.quotas.delete.assert_called_once_with(
             tenant_id)
+
+    def test_get(self):
+        tenant_id = "tenant_id"
+        quotas = {"gigabytes": "gb", "snapshots": "ss", "shares": "v",
+                  "snapshot_gigabytes": "sg", "share_networks": "sn"}
+        quota_set = mock.MagicMock(**quotas)
+        clients = mock.MagicMock()
+        clients.manila.return_value.quotas.get.return_value = quota_set
+        manila_quo = manila_quotas.ManilaQuotas(clients)
+
+        self.assertEqual(quotas, manila_quo.get(tenant_id))
+        clients.manila().quotas.get.assert_called_once_with(tenant_id)

tests/unit/plugins/openstack/context/quotas/test_neutron_quotas.py

@@ -14,18 +14,14 @@
 
 import mock
 
-from rally.plugins.openstack.context.quotas import neutron_quotas as quotas
+from rally.plugins.openstack.context.quotas import neutron_quotas
 from tests.unit import test
 
 
 class NeutronQuotasTestCase(test.TestCase):
-
+    def setUp(self):
-    @mock.patch("rally.plugins.openstack.context."
+        super(NeutronQuotasTestCase, self).setUp()
-                "quotas.quotas.osclients.Clients")
+        self.quotas = {
-    def test_update(self, mock_clients):
-        neutron_quotas = quotas.NeutronQuotas(mock_clients)
-        tenant_id = mock.MagicMock()
-        quotas_values = {
             "network": 20,
             "subnet": 20,
             "port": 100,
@@ -34,15 +30,29 @@ class NeutronQuotasTestCase(test.TestCase):
             "security_group": 100,
             "security_group_rule": 100
         }
-        neutron_quotas.update(tenant_id, **quotas_values)
+
-        body = {"quota": quotas_values}
+    def test_update(self):
-        mock_clients.neutron().update_quota.assert_called_once_with(tenant_id,
+        clients = mock.MagicMock()
+        neutron_quo = neutron_quotas.NeutronQuotas(clients)
+        tenant_id = mock.MagicMock()
+        neutron_quo.update(tenant_id, **self.quotas)
+        body = {"quota": self.quotas}
+        clients.neutron().update_quota.assert_called_once_with(tenant_id,
                                                                body=body)
 
-    @mock.patch("rally.plugins.openstack.context."
+    def test_delete(self):
-                "quotas.quotas.osclients.Clients")
+        clients = mock.MagicMock()
-    def test_delete(self, mock_clients):
+        neutron_quo = neutron_quotas.NeutronQuotas(clients)
-        neutron_quotas = quotas.NeutronQuotas(mock_clients)
         tenant_id = mock.MagicMock()
-        neutron_quotas.delete(tenant_id)
+        neutron_quo.delete(tenant_id)
-        mock_clients.neutron().delete_quota.assert_called_once_with(tenant_id)
+        clients.neutron().delete_quota.assert_called_once_with(tenant_id)
+
+    def test_get(self):
+        tenant_id = "tenant_id"
+        clients = mock.MagicMock()
+        clients.neutron.return_value.show_quota.return_value = {
+            "quota": self.quotas}
+        neutron_quo = neutron_quotas.NeutronQuotas(clients)
+
+        self.assertEqual(self.quotas, neutron_quo.get(tenant_id))
+        clients.neutron().show_quota.assert_called_once_with(tenant_id)

tests/unit/plugins/openstack/context/quotas/test_nova_quotas.py

@@ -14,18 +14,15 @@
 
 import mock
 
-from rally.plugins.openstack.context.quotas import nova_quotas as quotas
+from rally.plugins.openstack.context.quotas import nova_quotas
 from tests.unit import test
 
 
 class NovaQuotasTestCase(test.TestCase):
 
-    @mock.patch("rally.plugins.openstack.context."
+    def setUp(self):
-                "quotas.quotas.osclients.Clients")
+        super(NovaQuotasTestCase, self).setUp()
-    def test_update(self, mock_clients):
+        self.quotas = {
-        nova_quotas = quotas.NovaQuotas(mock_clients)
-        tenant_id = mock.MagicMock()
-        quotas_values = {
             "instances": 10,
             "cores": 100,
             "ram": 100000,
@@ -37,16 +34,32 @@ class NovaQuotasTestCase(test.TestCase):
             "injected_file_path_bytes": 1024,
             "key_pairs": 50,
             "security_groups": 50,
-            "security_group_rules": 50
+            "security_group_rules": 50,
+            "server_group_members": 777,
+            "server_groups": 33
         }
-        nova_quotas.update(tenant_id, **quotas_values)
-        mock_clients.nova().quotas.update.assert_called_once_with(
-            tenant_id, **quotas_values)
 
-    @mock.patch("rally.plugins.openstack.context."
+    def test_update(self):
-                "quotas.quotas.osclients.Clients")
+        clients = mock.MagicMock()
-    def test_delete(self, mock_clients):
+        nova_quo = nova_quotas.NovaQuotas(clients)
-        nova_quotas = quotas.NovaQuotas(mock_clients)
         tenant_id = mock.MagicMock()
-        nova_quotas.delete(tenant_id)
+        nova_quo.update(tenant_id, **self.quotas)
-        mock_clients.nova().quotas.delete.assert_called_once_with(tenant_id)
+        clients.nova().quotas.update.assert_called_once_with(tenant_id,
+                                                             **self.quotas)
+
+    def test_delete(self):
+        clients = mock.MagicMock()
+        nova_quo = nova_quotas.NovaQuotas(clients)
+        tenant_id = mock.MagicMock()
+        nova_quo.delete(tenant_id)
+        clients.nova().quotas.delete.assert_called_once_with(tenant_id)
+
+    def test_get(self):
+        tenant_id = "tenant_id"
+        quota_set = mock.MagicMock(**self.quotas)
+        clients = mock.MagicMock()
+        clients.nova.return_value.quotas.get.return_value = quota_set
+        nova_quo = nova_quotas.NovaQuotas(clients)
+
+        self.assertEqual(self.quotas, nova_quo.get(tenant_id))
+        clients.nova().quotas.get.assert_called_once_with(tenant_id)

tests/unit/plugins/openstack/context/quotas/test_quotas.py

@@ -20,10 +20,11 @@ import ddt
 import jsonschema
 import mock
 
+from rally.common import logging
 from rally.plugins.openstack.context.quotas import quotas
 from tests.unit import test
 
-QUOTAS_PATH = "rally.plugins.openstack.context.quotas."
+QUOTAS_PATH = "rally.plugins.openstack.context.quotas"
 
 
 @ddt.ddt
@@ -135,12 +136,14 @@ class QuotasTestCase(test.TestCase):
             except jsonschema.ValidationError:
                 self.fail("Valid quota keys are optional")
 
-    @mock.patch("rally.plugins.openstack.context."
+    @mock.patch("%s.quotas.osclients.Clients" % QUOTAS_PATH)
-                "quotas.quotas.osclients.Clients")
+    @mock.patch("%s.cinder_quotas.CinderQuotas" % QUOTAS_PATH)
-    @mock.patch("rally.plugins.openstack.context."
+    @ddt.data(True, False)
-                "quotas.cinder_quotas.CinderQuotas")
+    def test_cinder_quotas(self, ex_users, mock_cinder_quotas, mock_clients):
-    def test_cinder_quotas(self, mock_cinder_quotas, mock_clients):
+        cinder_quo = mock_cinder_quotas.return_value
         ctx = copy.deepcopy(self.context)
+        if ex_users:
+            ctx["existing_users"] = None
         ctx["config"]["quotas"] = {
             "cinder": {
                 "volumes": self.unlimited,
@@ -151,29 +154,33 @@ class QuotasTestCase(test.TestCase):
 
         tenants = ctx["tenants"]
         cinder_quotas = ctx["config"]["quotas"]["cinder"]
+        cinder_quo.get.return_value = cinder_quotas
         with quotas.Quotas(ctx) as quotas_ctx:
             quotas_ctx.setup()
-            expected_setup_calls = []
+            if ex_users:
-            for tenant in tenants:
+                self.assertEqual([mock.call(tenant) for tenant in tenants],
-                expected_setup_calls.append(mock.call()
+                                 cinder_quo.get.call_args_list)
-                                                .update(tenant,
+            self.assertEqual([mock.call(tenant, **cinder_quotas)
-                                                        **cinder_quotas))
+                              for tenant in tenants],
-            mock_cinder_quotas.assert_has_calls(
+                             cinder_quo.update.call_args_list)
-                expected_setup_calls, any_order=True)
             mock_cinder_quotas.reset_mock()
 
-        expected_cleanup_calls = []
+        if ex_users:
-        for tenant in tenants:
+            self.assertEqual([mock.call(tenant, **cinder_quotas)
-            expected_cleanup_calls.append(mock.call().delete(tenant))
+                              for tenant in tenants],
-        mock_cinder_quotas.assert_has_calls(
+                             cinder_quo.update.call_args_list)
-            expected_cleanup_calls, any_order=True)
+        else:
+            self.assertEqual([mock.call(tenant) for tenant in tenants],
+                             cinder_quo.delete.call_args_list)
 
-    @mock.patch("rally.plugins.openstack.context."
+    @mock.patch("%s.quotas.osclients.Clients" % QUOTAS_PATH)
-                "quotas.quotas.osclients.Clients")
+    @mock.patch("%s.nova_quotas.NovaQuotas" % QUOTAS_PATH)
-    @mock.patch("rally.plugins.openstack.context."
+    @ddt.data(True, False)
-                "quotas.nova_quotas.NovaQuotas")
+    def test_nova_quotas(self, ex_users, mock_nova_quotas, mock_clients):
-    def test_nova_quotas(self, mock_nova_quotas, mock_clients):
+        nova_quo = mock_nova_quotas.return_value
         ctx = copy.deepcopy(self.context)
+        if ex_users:
+            ctx["existing_users"] = None
 
         ctx["config"]["quotas"] = {
             "nova": {
@@ -192,30 +199,35 @@ class QuotasTestCase(test.TestCase):
             }
         }
 
+        tenants = ctx["tenants"]
         nova_quotas = ctx["config"]["quotas"]["nova"]
+        nova_quo.get.return_value = nova_quotas
         with quotas.Quotas(ctx) as quotas_ctx:
             quotas_ctx.setup()
-            expected_setup_calls = []
+            if ex_users:
-            for tenant in ctx["tenants"]:
+                self.assertEqual([mock.call(tenant) for tenant in tenants],
-                expected_setup_calls.append(mock.call()
+                                 nova_quo.get.call_args_list)
-                                                .update(tenant,
+            self.assertEqual([mock.call(tenant, **nova_quotas)
-                                                        **nova_quotas))
+                              for tenant in tenants],
-            mock_nova_quotas.assert_has_calls(
+                             nova_quo.update.call_args_list)
-                expected_setup_calls, any_order=True)
             mock_nova_quotas.reset_mock()
 
-        expected_cleanup_calls = []
+        if ex_users:
-        for tenant in ctx["tenants"]:
+            self.assertEqual([mock.call(tenant, **nova_quotas)
-            expected_cleanup_calls.append(mock.call().delete(tenant))
+                              for tenant in tenants],
-        mock_nova_quotas.assert_has_calls(
+                             nova_quo.update.call_args_list)
-            expected_cleanup_calls, any_order=True)
+        else:
+            self.assertEqual([mock.call(tenant) for tenant in tenants],
+                             nova_quo.delete.call_args_list)
 
-    @mock.patch("rally.plugins.openstack.context."
+    @mock.patch("%s.quotas.osclients.Clients" % QUOTAS_PATH)
-                "quotas.quotas.osclients.Clients")
+    @mock.patch("%s.neutron_quotas.NeutronQuotas" % QUOTAS_PATH)
-    @mock.patch("rally.plugins.openstack.context."
+    @ddt.data(True, False)
-                "quotas.neutron_quotas.NeutronQuotas")
+    def test_neutron_quotas(self, ex_users, mock_neutron_quotas, mock_clients):
-    def test_neutron_quotas(self, mock_neutron_quotas, mock_clients):
+        neutron_quo = mock_neutron_quotas.return_value
         ctx = copy.deepcopy(self.context)
+        if ex_users:
+            ctx["existing_users"] = None
 
         ctx["config"]["quotas"] = {
             "neutron": {
@@ -229,23 +241,26 @@ class QuotasTestCase(test.TestCase):
             }
         }
 
+        tenants = ctx["tenants"]
         neutron_quotas = ctx["config"]["quotas"]["neutron"]
+        neutron_quo.get.return_value = neutron_quotas
         with quotas.Quotas(ctx) as quotas_ctx:
             quotas_ctx.setup()
-            expected_setup_calls = []
+            if ex_users:
-            for tenant in ctx["tenants"]:
+                self.assertEqual([mock.call(tenant) for tenant in tenants],
-                expected_setup_calls.append(mock.call()
+                                 neutron_quo.get.call_args_list)
-                                                .update(tenant,
+            self.assertEqual([mock.call(tenant, **neutron_quotas)
-                                                        **neutron_quotas))
+                              for tenant in tenants],
-            mock_neutron_quotas.assert_has_calls(
+                             neutron_quo.update.call_args_list)
-                expected_setup_calls, any_order=True)
+            neutron_quo.reset_mock()
-            mock_neutron_quotas.reset_mock()
 
-        expected_cleanup_calls = []
+        if ex_users:
-        for tenant in ctx["tenants"]:
+            self.assertEqual([mock.call(tenant, **neutron_quotas)
-            expected_cleanup_calls.append(mock.call().delete(tenant))
+                              for tenant in tenants],
-        mock_neutron_quotas.assert_has_calls(
+                             neutron_quo.update.call_args_list)
-            expected_cleanup_calls, any_order=True)
+        else:
+            self.assertEqual([mock.call(tenant) for tenant in tenants],
+                             neutron_quo.delete.call_args_list)
 
     @mock.patch("rally.plugins.openstack.context."
                 "quotas.quotas.osclients.Clients")
@@ -285,15 +300,24 @@ class QuotasTestCase(test.TestCase):
     )
     @ddt.unpack
     def test_exception_during_cleanup(self, quotas_ctxt, quotas_class_path):
-        with mock.patch(QUOTAS_PATH + quotas_class_path) as mock_quotas:
+        quotas_path = "%s.%s" % (QUOTAS_PATH, quotas_class_path)
-            mock_quotas.delete.side_effect = type(
+        with mock.patch(quotas_path) as mock_quotas:
-                "ExceptionDuringCleanup", (Exception, ), {})
+            mock_quotas.return_value.update.side_effect = Exception
 
             ctx = copy.deepcopy(self.context)
             ctx["config"]["quotas"] = quotas_ctxt
 
+            quotas_instance = quotas.Quotas(ctx)
+            quotas_instance.original_quotas = []
+            for service in quotas_ctxt:
+                for tenant in self.context["tenants"]:
+                    quotas_instance.original_quotas.append(
+                        (service, tenant, quotas_ctxt[service]))
             # NOTE(boris-42): ensure that cleanup didn't raise exceptions.
-            quotas.Quotas(ctx).cleanup()
+            with logging.LogCatcher(quotas.LOG) as log:
+                quotas_instance.cleanup()
 
-            self.assertEqual(mock_quotas.return_value.delete.call_count,
+                log.assertInLogs("Failed to restore quotas for tenant")
+
+            self.assertEqual(mock_quotas.return_value.update.call_count,
                              len(self.context["tenants"]))