Support quota API

Currently Aodh only supports global quota for projects and users, we
need quota API to manage quotas per project.

The new quota API only supports project quotas as the same with other
openstack services.

By default, only admin user can update the quota.

TODO(in subsequent patches):
- Check quota from DB when creating resources
- Doc
- Release note
- openstack CLI support

Change-Id: I9258ae4801edbc5289d890fe2e060964a73b216c

aodh/api/controllers/v2/alarms.py

@@ -51,14 +51,15 @@ from aodh.storage import models
 
 LOG = log.getLogger(__name__)
 
-
 ALARM_API_OPTS = [
     cfg.IntOpt('user_alarm_quota',
                deprecated_group='DEFAULT',
+               default=-1,
                help='Maximum number of alarms defined for a user.'
                ),
     cfg.IntOpt('project_alarm_quota',
                deprecated_group='DEFAULT',
+               default=-1,
                help='Maximum number of alarms defined for a project.'
                ),
     cfg.IntOpt('alarm_max_actions',
@@ -106,14 +107,14 @@ def is_over_quota(conn, project_id, user_id):
 
     # Start by checking for user quota
     user_alarm_quota = pecan.request.cfg.api.user_alarm_quota
-    if user_alarm_quota is not None:
+    if user_alarm_quota != -1:
         user_alarms = conn.get_alarms(user_id=user_id)
         over_quota = len(user_alarms) >= user_alarm_quota
 
     # If the user quota isn't reached, we check for the project quota
     if not over_quota:
         project_alarm_quota = pecan.request.cfg.api.project_alarm_quota
-        if project_alarm_quota is not None:
+        if project_alarm_quota != -1:
             project_alarms = conn.get_alarms(project_id=project_id)
             over_quota = len(project_alarms) >= project_alarm_quota
 

aodh/api/controllers/v2/base.py

@@ -77,6 +77,7 @@ class AdvEnum(wtypes.wsproperty):
 
 
 class Base(wtypes.DynamicBase):
+    _wsme_attributes = []
 
     @classmethod
     def from_db_model(cls, m):
@@ -98,6 +99,14 @@ class Base(wtypes.DynamicBase):
                     if hasattr(self, k) and
                     getattr(self, k) != wsme.Unset)
 
+    def to_dict(self):
+        d = {}
+        for attr in self._wsme_attributes:
+            attr_val = getattr(self, attr.name)
+            if not isinstance(attr_val, wtypes.UnsetType):
+                d[attr.name] = attr_val
+        return d
+
 
 class Query(Base):
     """Query filter."""

aodh/api/controllers/v2/quotas.py

@@ -0,0 +1,84 @@
+# Copyright 2020 Catalyst Cloud LTD.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+from oslo_log import log
+import pecan
+from pecan import rest
+import wsme
+from wsme import types as wtypes
+import wsmeext.pecan as wsme_pecan
+
+from aodh.api.controllers.v2 import base
+from aodh.api import rbac
+
+LOG = log.getLogger(__name__)
+ALLOWED_RESOURCES = ('alarms',)
+
+
+class Quota(base.Base):
+    resource = wtypes.wsattr(wtypes.Enum(str, *ALLOWED_RESOURCES),
+                             mandatory=True)
+    limit = wtypes.IntegerType(minimum=-1)
+
+
+class Quotas(base.Base):
+    project_id = wsme.wsattr(wtypes.text, mandatory=True)
+    quotas = [Quota]
+
+
+class QuotasController(rest.RestController):
+    """Quota API controller."""
+
+    @wsme_pecan.wsexpose(Quotas, str, ignore_extra_args=True)
+    def get_all(self, project_id=None):
+        """Get resource quotas of a project.
+
+        - If no project given, get requested user's quota.
+        - Admin user can get resource quotas of any project.
+        """
+        request_project = pecan.request.headers.get('X-Project-Id')
+        project_id = project_id if project_id else request_project
+        is_admin = rbac.is_admin(pecan.request.headers)
+
+        if project_id != request_project and not is_admin:
+            raise base.ProjectNotAuthorized(project_id)
+
+        LOG.debug('Getting resource quotas for project %s', project_id)
+
+        db_quotas = pecan.request.storage.get_quotas(project_id=project_id)
+
+        if len(db_quotas) == 0:
+            project_alarm_quota = pecan.request.cfg.api.project_alarm_quota
+            quotas = [{'resource': 'alarms', 'limit': project_alarm_quota}]
+            db_quotas = pecan.request.storage.set_quotas(project_id, quotas)
+
+        quotas = [Quota.from_db_model(i) for i in db_quotas]
+        return Quotas(project_id=project_id, quotas=quotas)
+
+    @wsme_pecan.wsexpose(Quotas, body=Quotas, status_code=201)
+    def post(self, body):
+        """Create or update quota."""
+        rbac.enforce('update_quotas', pecan.request.headers,
+                     pecan.request.enforcer, {})
+
+        params = body.to_dict()
+        project_id = params['project_id']
+
+        input_quotas = []
+        for i in params.get('quotas', []):
+            input_quotas.append(i.to_dict())
+
+        db_quotas = pecan.request.storage.set_quotas(project_id, input_quotas)
+
+        quotas = [Quota.from_db_model(i) for i in db_quotas]
+        return Quotas(project_id=project_id, quotas=quotas)

aodh/api/controllers/v2/root.py

@@ -21,6 +21,7 @@
 from aodh.api.controllers.v2 import alarms
 from aodh.api.controllers.v2 import capabilities
 from aodh.api.controllers.v2 import query
+from aodh.api.controllers.v2 import quotas
 
 
 class V2Controller(object):
@@ -29,3 +30,4 @@ class V2Controller(object):
     alarms = alarms.AlarmsController()
     query = query.QueryController()
     capabilities = capabilities.CapabilitiesController()
+    quotas = quotas.QuotasController()

aodh/api/policies.py

@@ -155,6 +155,17 @@ rules = [
                 'method': 'POST'
             }
         ]
+    ),
+    policy.DocumentedRuleDefault(
+        name="telemetry:update_quotas",
+        check_str=RULE_CONTEXT_IS_ADMIN,
+        description='Update resources quotas for project.',
+        operations=[
+            {
+                'path': '/v2/quotas',
+                'method': 'POST'
+            }
+        ]
     )
 ]
 

aodh/api/rbac.py

@@ -54,13 +54,6 @@ def enforce(policy_name, headers, enforcer, target):
         'project_id': headers.get('X-Project-Id'),
     }
 
-    # TODO(sileht): add deprecation warning to be able to remove this:
-    # maintain backward compat with Juno and previous by allowing the action if
-    # there is no rule defined for it
-    rules = enforcer.rules.keys()
-    if rule_method not in rules:
-        return
-
     if not enforcer.enforce(rule_method, target, credentials):
         pecan.core.abort(status_code=403,
                          detail='RBAC Authorization Failed')

aodh/storage/base.py

@@ -203,3 +203,15 @@ class Connection(object):
         """
         raise aodh.NotImplementedError('Clearing alarm history '
                                        'not implemented')
+
+    @staticmethod
+    def get_quotas(project_id):
+        """Get resource quota for the given project."""
+        raise aodh.NotImplementedError('Getting resource quota not '
+                                       'implemented')
+
+    @staticmethod
+    def set_quotas(project_id, quotas):
+        """Set resource quota for the given user."""
+        raise aodh.NotImplementedError('Setting resource quota not '
+                                       'implemented')

aodh/storage/impl_sqlalchemy.py

@@ -432,3 +432,46 @@ class Connection(base.Connection):
 
         result = query.update(values, **update_args)
         return 0 != result
+
+    @staticmethod
+    def _row_to_quota_model(row):
+        return alarm_api_models.Quota(
+            project_id=row.project_id,
+            resource=row.resource,
+            limit=row.limit,
+        )
+
+    def _retrieve_quotas(self, query):
+        return [self._row_to_quota_model(x) for x in query.all()]
+
+    def get_quotas(self, project_id):
+        """Get resource quota for the given project."""
+        filters = {'project_id': project_id}
+        session = self._engine_facade.get_session()
+        query = session.query(models.Quota).filter_by(**filters)
+        return self._retrieve_quotas(query)
+
+    def set_quotas(self, project_id, quotas):
+        """Set resource quota for the given user."""
+        session = self._engine_facade.get_session()
+
+        with session.begin():
+            for q in quotas:
+                values = {
+                    'project_id': project_id,
+                    'resource': q['resource'],
+                }
+
+                quota = session.query(models.Quota).filter_by(**values).first()
+                if not quota:
+                    new_quota = models.Quota(project_id=project_id,
+                                             resource=q['resource'],
+                                             limit=q['limit'])
+                    session.add(new_quota)
+                else:
+                    values['limit'] = q['limit']
+                    quota.update(values.copy())
+
+        filters = {'project_id': project_id}
+        query = session.query(models.Quota).filter_by(**filters)
+        return self._retrieve_quotas(query)

aodh/storage/models.py

@@ -151,3 +151,12 @@ class AlarmChange(base.Model):
             project_id=project_id,
             on_behalf_of=on_behalf_of,
             timestamp=timestamp)
+
+
+class Quota(base.Model):
+    def __init__(self, project_id, resource, limit):
+        base.Model.__init__(
+            self,
+            project_id=project_id,
+            resource=resource,
+            limit=limit)

aodh/storage/sqlalchemy/alembic/versions/007_add_quota_table.py

@@ -0,0 +1,43 @@
+# Copyright 2020 Catalyst Cloud Ltd.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+"""Add quota table
+
+Revision ID: 007
+Revises: 006
+Create Date: 2020-01-28
+"""
+
+# revision identifiers, used by Alembic.
+revision = '007'
+down_revision = '006'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.create_table(
+        'quota',
+        sa.Column('id', sa.String(length=36), nullable=False),
+        sa.Column('resource', sa.String(length=50), nullable=False),
+        sa.Column('project_id', sa.String(length=128), nullable=False),
+        sa.Column('limit', sa.Integer, nullable=False),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('project_id', 'resource'),
+        sa.Index(
+            'ix_quota_project_id_resource',
+            'project_id', 'resource'
+        )
+    )

aodh/storage/sqlalchemy/models.py

@@ -16,8 +16,10 @@ SQLAlchemy models for aodh data.
 import json
 
 from oslo_utils import timeutils
+from oslo_utils import uuidutils
 import six
-from sqlalchemy import Column, String, Index, Boolean, Text, DateTime
+import sqlalchemy as sa
+from sqlalchemy import Column, String, Index, Boolean, Text, DateTime, Integer
 from sqlalchemy.dialects import mysql
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.types import TypeDecorator
@@ -124,3 +126,17 @@ class AlarmChange(Base):
     detail = Column(Text)
     timestamp = Column(TimestampUTC, default=lambda: timeutils.utcnow())
     severity = Column(String(50))
+
+
+class Quota(Base):
+    __tablename__ = 'quota'
+    __table_args__ = (
+        sa.UniqueConstraint('project_id', 'resource'),
+        Index('ix_%s_project_id_resource' % __tablename__,
+              'project_id', 'resource'),
+    )
+
+    id = Column(String(36), primary_key=True, default=uuidutils.generate_uuid)
+    project_id = Column(String(128))
+    resource = Column(String(50))
+    limit = Column(Integer)

aodh/tests/base.py

@@ -91,6 +91,29 @@ class BaseTestCase(base.BaseTestCase):
         else:
             return root
 
+    def assert_single_item(self, items, **filters):
+        return self.assert_multiple_items(items, 1, **filters)[0]
+
+    def assert_multiple_items(self, items, count, **filters):
+        def _matches(item, **props):
+            for prop_name, prop_val in props.items():
+                v = (item[prop_name] if isinstance(item, dict)
+                     else getattr(item, prop_name))
+                if v != prop_val:
+                    return False
+            return True
+
+        filtered_items = list(
+            [item for item in items if _matches(item, **filters)]
+        )
+        found = len(filtered_items)
+
+        if found != count:
+            self.fail("Wrong number of items found [filters=%s, "
+                      "expected=%s, found=%s]" % (filters, count, found))
+
+        return filtered_items
+
 
 def _skip_decorator(func):
     @functools.wraps(func)

aodh/tests/functional/api/v2/test_quotas.py

@@ -0,0 +1,102 @@
+# Copyright 2020 Catalyst Cloud LTD.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+import copy
+
+from oslo_utils import uuidutils
+
+from aodh.tests.functional.api import v2
+
+
+class TestQuotas(v2.FunctionalTest):
+    @classmethod
+    def setUpClass(cls):
+        super(TestQuotas, cls).setUpClass()
+
+        cls.project = uuidutils.generate_uuid()
+        cls.user = uuidutils.generate_uuid()
+        cls.auth_headers = {'X-User-Id': cls.user, 'X-Project-Id': cls.project}
+        cls.other_project = uuidutils.generate_uuid()
+
+    def test_get_quotas_by_user(self):
+        resp = self.get_json('/quotas', headers=self.auth_headers, status=200)
+
+        self.assertEqual(self.project, resp.get('project_id'))
+        self.assertTrue(len(resp.get('quotas', [])) > 0)
+
+    def test_get_project_quotas_by_user(self):
+        resp = self.get_json('/quotas?project_id=%s' % self.project,
+                             headers=self.auth_headers, status=200)
+
+        self.assertEqual(self.project, resp.get('project_id'))
+        self.assertTrue(len(resp.get('quotas', [])) > 0)
+
+    def test_get_other_project_quotas_by_user_failed(self):
+        self.get_json(
+            '/quotas?project_id=%s' % self.other_project,
+            headers=self.auth_headers,
+            expect_errors=True,
+            status=401
+        )
+
+    def test_get_project_quotas_by_admin(self):
+        auth_headers = copy.copy(self.auth_headers)
+        auth_headers['X-Roles'] = 'admin'
+
+        resp = self.get_json('/quotas?project_id=%s' % self.other_project,
+                             headers=auth_headers,
+                             status=200)
+
+        self.assertEqual(self.other_project, resp.get('project_id'))
+        self.assertTrue(len(resp.get('quotas', [])) > 0)
+
+    def test_post_quotas_by_admin(self):
+        auth_headers = copy.copy(self.auth_headers)
+        auth_headers['X-Roles'] = 'admin'
+
+        resp = self.post_json(
+            '/quotas',
+            {
+                "project_id": self.other_project,
+                "quotas": [
+                    {
+                        "resource": "alarms",
+                        "limit": 30
+                    }
+                ]
+            },
+            headers=auth_headers,
+            status=201
+        )
+        resp_json = resp.json
+
+        self.assertEqual(self.other_project, resp_json.get('project_id'))
+        self.assert_single_item(resp_json.get('quotas', []), resource='alarms',
+                                limit=30)
+
+    def test_post_quotas_by_user_failed(self):
+        self.post_json(
+            '/quotas',
+            {
+                "project_id": self.other_project,
+                "quotas": [
+                    {
+                        "resource": "alarms",
+                        "limit": 20
+                    }
+                ]
+            },
+            headers=self.auth_headers,
+            expect_errors=True,
+            status=403
+        )