Move more quota logic into QuotaManager

* Makes it easier to add more services to the quota actions
* Fixes issues with endpoints that don't exist
* Allows deployers to override the services using the quotas
  available

Change-Id: Iff64d33a7f3773d5c9b1674c3dccb4804804b0a0

adjutant/actions/v1/resources.py

@@ -235,37 +235,6 @@ class UpdateProjectQuotasAction(BaseAction, QuotaMixin):
 
     default_days_between_autoapprove = 30
 
-    class ServiceQuotaFunctor(object):
-        def __call__(self, project_id, values):
-            self.client.quotas.update(project_id, **values)
-
-    class ServiceQuotaCinderFunctor(ServiceQuotaFunctor):
-        def __init__(self, region_name):
-            self.client = openstack_clients.get_cinderclient(
-                region=region_name)
-
-    class ServiceQuotaNovaFunctor(ServiceQuotaFunctor):
-        def __init__(self, region_name):
-            self.client = openstack_clients.get_novaclient(
-                region=region_name)
-
-    class ServiceQuotaNeutronFunctor(ServiceQuotaFunctor):
-        def __init__(self, region_name):
-            self.client = openstack_clients.get_neutronclient(
-                region=region_name)
-
-        def __call__(self, project_id, values):
-            body = {
-                'quota': values
-            }
-            self.client.update_quota(project_id, body)
-
-    _quota_updaters = {
-        'cinder': ServiceQuotaCinderFunctor,
-        'nova': ServiceQuotaNovaFunctor,
-        'neutron': ServiceQuotaNeutronFunctor
-    }
-
     def __init__(self, *args, **kwargs):
         super(UpdateProjectQuotasAction, self).__init__(*args, **kwargs)
         self.size_difference_threshold = settings.TASK_SETTINGS.get(
@@ -302,15 +271,11 @@ class UpdateProjectQuotasAction(BaseAction, QuotaMixin):
                     quota_size, region_name))
             return
 
-        for service_name, values in quota_settings.items():
+        quota_manager = QuotaManager(self.project_id,
-            updater_class = self._quota_updaters.get(service_name)
+                                     self.size_difference_threshold)
-            if not updater_class:
+
-                self.add_note("No quota updater found for %s. Ignoring" %
+        quota_manager.set_region_quota(region_name, quota_settings)
-                              service_name)
+
-                continue
-            # functor for the service+region
-            service_functor = updater_class(region_name)
-            service_functor(self.project_id, values)
         self.add_note("Project quota for region %s set to %s" % (
                       region_name, quota_size))
 
@@ -387,6 +352,8 @@ class UpdateProjectQuotasAction(BaseAction, QuotaMixin):
         if not self.valid or self.action.state == "completed":
             return
 
+        # Use manager here instead, it will make it easier to add has_more
+        # in later
         for region in self.regions:
             self._set_region_quota(region, self.size)
 

adjutant/actions/v1/tests/test_resource_actions.py

@@ -35,12 +35,13 @@ from adjutant.common.tests.fake_clients import (
     'openstack_clients.get_neutronclient',
     get_fake_neutron)
 @mock.patch(
-    'adjutant.actions.v1.resources.' +
+    'adjutant.common.quota.get_neutronclient',
-    'openstack_clients.get_novaclient',
+    get_fake_neutron)
+@mock.patch(
+    'adjutant.common.quota.get_novaclient',
     get_fake_novaclient)
 @mock.patch(
-    'adjutant.actions.v1.resources.' +
+    'adjutant.common.quota.get_cinderclient',
-    'openstack_clients.get_cinderclient',
     get_fake_cinderclient)
 class ProjectSetupActionTests(TestCase):
 
@@ -475,18 +476,6 @@ class ProjectSetupActionTests(TestCase):
 @mock.patch(
     'adjutant.common.quota.get_cinderclient',
     get_fake_cinderclient)
-@mock.patch(
-    'adjutant.actions.v1.resources.' +
-    'openstack_clients.get_neutronclient',
-    get_fake_neutron)
-@mock.patch(
-    'adjutant.actions.v1.resources.' +
-    'openstack_clients.get_novaclient',
-    get_fake_novaclient)
-@mock.patch(
-    'adjutant.actions.v1.resources.' +
-    'openstack_clients.get_cinderclient',
-    get_fake_cinderclient)
 class QuotaActionTests(TestCase):
 
     def test_update_quota(self):

adjutant/common/quota.py

@@ -27,22 +27,133 @@ class QuotaManager(object):
 
     default_size_diff_threshold = .2
 
+    class ServiceQuotaHelper(object):
+        def set_quota(self, values):
+            self.client.quotas.update(self.project_id, **values)
+
+    class ServiceQuotaCinderHelper(ServiceQuotaHelper):
+        def __init__(self, region_name, project_id):
+            self.client = get_cinderclient(
+                region=region_name)
+            self.project_id = project_id
+
+        def get_quota(self):
+            return self.client.quotas.get(self.project_id).to_dict()
+
+        def get_usage(self):
+            volumes = self.client.volumes.list(
+                search_opts={'all_tenants': 1, 'project_id': self.project_id})
+            snapshots = self.client.volume_snapshots.list(
+                search_opts={'all_tenants': 1, 'project_id': self.project_id})
+
+            # gigabytesUsed should be a total of volumes and snapshots
+            gigabytes = sum([getattr(volume, 'size', 0) for volume
+                            in volumes])
+            gigabytes += sum([getattr(snap, 'size', 0) for snap
+                             in snapshots])
+
+            return {'gigabytes': gigabytes,
+                    'volumes': len(volumes),
+                    'snapshots': len(snapshots)
+                    }
+
+    class ServiceQuotaNovaHelper(ServiceQuotaHelper):
+        def __init__(self, region_name, project_id):
+            self.client = get_novaclient(
+                region=region_name)
+            self.project_id = project_id
+
+        def get_quota(self):
+            return self.client.quotas.get(self.project_id).to_dict()
+
+        def get_usage(self):
+            nova_usage = self.client.limits.get(
+                tenant_id=self.project_id).to_dict()['absolute']
+            nova_usage_keys = [
+                ('instances', 'totalInstancesUsed'),
+                ('floating_ips', 'totalFloatingIpsUsed'),
+                ('ram', 'totalRAMUsed'),
+                ('cores', 'totalCoresUsed'),
+                ('secuirty_groups', 'totalSecurityGroupsUsed')
+            ]
+
+            nova_usage_dict = {}
+            for key, usage_key in nova_usage_keys:
+                nova_usage_dict[key] = nova_usage[usage_key]
+
+            return nova_usage_dict
+
+    class ServiceQuotaNeutronHelper(ServiceQuotaHelper):
+        def __init__(self, region_name, project_id):
+            self.client = get_neutronclient(
+                region=region_name)
+            self.project_id = project_id
+
+        def set_quota(self, values):
+            body = {
+                'quota': values
+            }
+            self.client.update_quota(self.project_id, body)
+
+        def get_usage(self):
+            networks = self.client.list_networks(
+                tenant_id=self.project_id)['networks']
+            routers = self.client.list_routers(
+                tenant_id=self.project_id)['routers']
+            floatingips = self.client.list_floatingips(
+                tenant_id=self.project_id)['floatingips']
+            ports = self.client.list_ports(
+                tenant_id=self.project_id)['ports']
+            subnets = self.client.list_subnets(
+                tenant_id=self.project_id)['subnets']
+            security_groups = self.client.list_security_groups(
+                tenant_id=self.project_id)['security_groups']
+            security_group_rules = self.client.list_security_group_rules(
+                tenant_id=self.project_id)['security_group_rules']
+
+            return {'network': len(networks),
+                    'router': len(routers),
+                    'floatingip': len(floatingips),
+                    'port': len(ports),
+                    'subnet': len(subnets),
+                    'secuirty_group': len(security_groups),
+                    'security_group_rule': len(security_group_rules)
+                    }
+
+        def get_quota(self):
+            return self.client.show_quota(self.project_id)['quota']
+
+    _quota_updaters = {
+        'cinder': ServiceQuotaCinderHelper,
+        'nova': ServiceQuotaNovaHelper,
+        'neutron': ServiceQuotaNeutronHelper
+    }
+
     def __init__(self, project_id, size_difference_threshold=None):
+        # TODO(amelia): Try to find out which endpoints are available and get
+        # the non enabled ones out of the list
+
+        # Check configured removal of quota updaters
+        self.helpers = dict(self._quota_updaters)
+
+        # Configurable services
+        if settings.QUOTA_SERVICES:
+            self.helpers = {}
+            for name in settings.QUOTA_SERVICES:
+                if name in self._quota_updaters:
+                    self.helpers[name] = self._quota_updaters[name]
+
         self.project_id = project_id
         self.size_diff_threshold = (size_difference_threshold or
                                     self.default_size_diff_threshold)
 
     def get_current_region_quota(self, region_id):
-        ci_quota = get_cinderclient(region_id) \
+        current_quota = {}
-            .quotas.get(self.project_id).to_dict()
+        for name, service in self.helpers.items():
-        neutron_quota = get_neutronclient(region_id) \
+            helper = service(region_id, self.project_id)
-            .show_quota(self.project_id)['quota']
+            current_quota[name] = helper.get_quota()
-        nova_quota = get_novaclient(region_id) \
-            .quotas.get(self.project_id).to_dict()
 
-        return {'cinder': ci_quota,
+        return current_quota
-                'nova': nova_quota,
-                'neutron': neutron_quota}
 
     def get_quota_size(self, current_quota, difference_threshold=None):
         """ Gets the closest matching quota size for a given quota """
@@ -104,73 +215,34 @@ class QuotaManager(object):
                 }
 
     def get_current_usage(self, region_id):
-        cinder_usage = self.get_cinder_usage(region_id)
+        current_usage = {}
-        nova_usage = self.get_nova_usage(region_id)
-        neutron_usage = self.get_neutron_usage(region_id)
-        return {'cinder': cinder_usage,
-                'nova': nova_usage,
-                'neutron': neutron_usage}
 
-    def get_cinder_usage(self, region_id):
+        for name, service in self.helpers.items():
-        client = get_cinderclient(region_id)
+            try:
+                helper = service(region_id, self.project_id)
+                current_usage[name] = helper.get_usage()
+            except Exception:
+                pass
+        return current_usage
 
-        volumes = client.volumes.list(
+    def set_region_quota(self, region_id, quota_dict):
-            search_opts={'all_tenants': 1, 'project_id': self.project_id})
+        notes = []
-        snapshots = client.volume_snapshots.list(
+        for service_name, values in quota_dict.items():
-            search_opts={'all_tenants': 1, 'project_id': self.project_id})
+            updater_class = self.helpers.get(service_name)
+            if not updater_class:
+                notes.append("No quota updater found for %s. Ignoring" %
+                             service_name)
+                continue
 
-        # gigabytesUsed should be a total of volumes and snapshots
+            try:
-        gigabytes = sum([getattr(volume, 'size', 0) for volume
+                service_helper = updater_class(region_id, self.project_id)
-                        in volumes])
+            except Exception:
-        gigabytes += sum([getattr(snap, 'size', 0) for snap
+                # NOTE(amelia): We will assume if there are issues connecting
-                         in snapshots])
+                #               to a service that it will be due to the
+                #               service not existing in this region.
+                notes.append("Couldn't access %s client, region %s" %
+                             (service_name, region_id))
+                continue
 
-        return {'gigabytes': gigabytes,
+            service_helper.set_quota(values)
-                'volumes': len(volumes),
+        return notes
-                'snapshots': len(snapshots)
-                }
-
-    def get_neutron_usage(self, region_id):
-        client = get_neutronclient(region_id)
-
-        networks = client.list_networks(
-            tenant_id=self.project_id)['networks']
-        routers = client.list_routers(
-            tenant_id=self.project_id)['routers']
-        floatingips = client.list_floatingips(
-            tenant_id=self.project_id)['floatingips']
-        ports = client.list_ports(
-            tenant_id=self.project_id)['ports']
-        subnets = client.list_subnets(
-            tenant_id=self.project_id)['subnets']
-        security_groups = client.list_security_groups(
-            tenant_id=self.project_id)['security_groups']
-        security_group_rules = client.list_security_group_rules(
-            tenant_id=self.project_id)['security_group_rules']
-
-        return {'network': len(networks),
-                'router': len(routers),
-                'floatingip': len(floatingips),
-                'port': len(ports),
-                'subnet': len(subnets),
-                'secuirty_group': len(security_groups),
-                'security_group_rule': len(security_group_rules)
-                }
-
-    def get_nova_usage(self, region_id):
-        client = get_novaclient(region_id)
-        nova_usage = client.limits.get(
-            tenant_id=self.project_id).to_dict()['absolute']
-        nova_usage_keys = [
-            ('instances', 'totalInstancesUsed'),
-            ('floating_ips', 'totalFloatingIpsUsed'),
-            ('ram', 'totalRAMUsed'),
-            ('cores', 'totalCoresUsed'),
-            ('secuirty_groups', 'totalSecurityGroupsUsed')
-        ]
-
-        nova_usage_dict = {}
-        for key, usage_key in nova_usage_keys:
-            nova_usage_dict[key] = nova_usage[usage_key]
-
-        return nova_usage_dict

adjutant/settings.py

@@ -191,6 +191,10 @@ ACTIVE_TASKVIEWS = CONFIG.get(
         'RoleList'
     ])
 
+# Default services for which to check and update quotas for
+QUOTA_SERVICES = CONFIG.get('QUOTA_SERVICES',
+                            ['cinder', 'neutron', 'nova'])
+
 # Dict of TaskViews and their url_paths.
 # - This is populated by registering taskviews.
 TASKVIEW_CLASSES = {}

adjutant/test_settings.py

@@ -384,6 +384,8 @@ PROJECT_QUOTA_SIZES = {
 
 QUOTA_SIZES_ASC = ['small', 'medium', 'large']
 
+QUOTA_SERVICES = ['neutron', 'nova', 'cinder']
+
 SHOW_ACTION_ENDPOINTS = True
 
 TOKEN_CACHE_TIME = 60
@@ -408,4 +410,5 @@ conf_dict = {
     "SHOW_ACTION_ENDPOINTS": SHOW_ACTION_ENDPOINTS,
     "QUOTA_SIZES_ASC": QUOTA_SIZES_ASC,
     "TOKEN_CACHE_TIME": TOKEN_CACHE_TIME,
+    "QUOTA_SERVICES": QUOTA_SERVICES,
 }

conf/conf.yaml

@@ -418,3 +418,9 @@ QUOTA_SIZES_ASC:
     - small
     - medium
     - large
+
+# Services to check through the quotas for
+QUOTA_SERVICES:
+    - nova
+    - neutron
+    - cinder