Add API support for flavors

This patch adds the control API for flavors. The patch allows users to
create, delete, update and list flavors. This API is considered an
admin-only API, hence it's registered as part of the private endpoints.

DocImpact

Partially-Implements blueprint marconi-queue-flavors

Change-Id: Id29d29940f2ecabab0531edefe018c0dd2f39811

tests/unit/queues/transport/wsgi/test_v1_1.py

@@ -97,10 +97,14 @@ class TestValidation(v1_1.TestValidation):
     url_prefix = URL_PREFIX
 
 
+class TestFlavorsMongoDB(v1_1.TestFlavorsMongoDB):
+    url_prefix = URL_PREFIX
+
 # --------------------------------------------------------------------------
 # v1.1 only
 # --------------------------------------------------------------------------
 
+
 class TestPing(base.V1_1Base):
 
     config_file = 'wsgi_sqlalchemy.conf'

zaqar/common/schemas/flavors.py

@@ -0,0 +1,51 @@
+# Copyright (c) 2013 Rackspace Hosting, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""flavors: JSON schema for marconi-queues flavors resources."""
+
+# NOTE(flaper87): capabilities can be anything. These will be unique to
+# each storage driver, so we don't perform any further validation at
+# the transport layer.
+patch_capabilities = {
+    'type': 'object',
+    'properties': {
+        'capabilities': {
+            'type': 'object'
+        }
+    }
+}
+
+# NOTE(flaper87): a string valid
+patch_pool = {
+    'type': 'object',
+    'properties': {
+        'pool': {
+            'type': 'string'
+        },
+        'additionalProperties': False
+    }
+}
+
+create = {
+    'type': 'object',
+    'properties': {
+        'pool': patch_pool['properties']['pool'],
+        'capabilities': patch_capabilities['properties']['capabilities']
+    },
+    # NOTE(flaper87): capabilities need not be present. Storage drivers
+    # must provide reasonable defaults.
+    'required': ['pool'],
+    'additionalProperties': False
+}

zaqar/queues/storage/mongodb/flavors.py

@@ -58,7 +58,7 @@ class FlavorsController(base.FlavorsBase):
             query['n'] = {'$gt': marker}
 
         cursor = self._col.find(query, fields=_field_spec(detailed),
-                                limit=limit)
+                                limit=limit).sort('n', 1)
 
         normalizer = functools.partial(_normalize, detailed=detailed)
         return utils.HookedCursor(cursor, normalizer)

zaqar/queues/storage/sqlalchemy/driver.py

@@ -181,4 +181,4 @@ class ControlDriver(storage.ControlDriverBase):
     @property
     def flavors_controller(self):
         # NOTE(flaper87): Needed to avoid `abc` errors.
-        raise NotImplementedError
+        pass

zaqar/queues/transport/wsgi/v1_1/__init__.py

@@ -13,6 +13,7 @@
 # the License.
 
 from zaqar.queues.transport.wsgi.v1_1 import claims
+from zaqar.queues.transport.wsgi.v1_1 import flavors
 from zaqar.queues.transport.wsgi.v1_1 import health
 from zaqar.queues.transport.wsgi.v1_1 import homedoc
 from zaqar.queues.transport.wsgi.v1_1 import messages
@@ -75,6 +76,7 @@ def public_endpoints(driver):
 
 def private_endpoints(driver):
     pools_controller = driver._control.pools_controller
+    flavors_controller = driver._control.flavors_controller
 
     return [
         ('/pools',
@@ -84,4 +86,8 @@ def private_endpoints(driver):
         # Health
         ('/health',
          health.Resource(driver._storage)),
+        ('/flavors',
+         flavors.Listing(flavors_controller)),
+        ('/flavors/{flavor}',
+         flavors.Resource(flavors_controller)),
     ]

zaqar/queues/transport/wsgi/v1_1/flavors.py

@@ -0,0 +1,185 @@
+# Copyright (c) 2014 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import falcon
+import jsonschema
+
+from zaqar.common.schemas import flavors as schema
+from zaqar.common import utils as common_utils
+from zaqar.openstack.common import log
+from zaqar.queues.storage import errors
+from zaqar.queues.transport import utils as transport_utils
+from zaqar.queues.transport.wsgi import errors as wsgi_errors
+from zaqar.queues.transport.wsgi import utils as wsgi_utils
+
+LOG = log.getLogger(__name__)
+
+
+class Listing(object):
+    """A resource to list registered flavors
+
+    :param flavors_controller: means to interact with storage
+    """
+
+    def __init__(self, flavors_controller):
+        self._ctrl = flavors_controller
+
+    def on_get(self, request, response, project_id):
+        """Returns a flavor listing as objects embedded in an array:
+
+        [
+            {"href": "", "capabilities": {}, "pool": ""},
+            ...
+        ]
+
+        :returns: HTTP | [200, 204]
+
+        """
+
+        LOG.debug(u'LIST flavors for project_id %s' % project_id)
+
+        store = {}
+        request.get_param('marker', store=store)
+        request.get_param_as_int('limit', store=store)
+        request.get_param_as_bool('detailed', store=store)
+
+        results = {}
+        results['flavors'] = list(self._ctrl.list(project=project_id, **store))
+        for entry in results['flavors']:
+            entry['href'] = request.path + '/' + entry.pop('name')
+
+        if not results['flavors']:
+            response.status = falcon.HTTP_204
+            return
+
+        response.content_location = request.relative_uri
+        response.body = transport_utils.to_json(results)
+        response.status = falcon.HTTP_200
+
+
+class Resource(object):
+    """A handler for individual flavor.
+
+    :param flavors_controller: means to interact with storage
+    """
+
+    def __init__(self, flavors_controller):
+        self._ctrl = flavors_controller
+        validator_type = jsonschema.Draft4Validator
+        self._validators = {
+            'create': validator_type(schema.create),
+            'pool': validator_type(schema.patch_pool),
+            'capabilities': validator_type(schema.patch_capabilities),
+        }
+
+    def on_get(self, request, response, project_id, flavor):
+        """Returns a JSON object for a single flavor entry:
+
+        {"pool": "", capabilities: {...}}
+
+        :returns: HTTP | [200, 404]
+
+        """
+
+        LOG.debug(u'GET flavor - name: %s', flavor)
+        data = None
+        detailed = request.get_param_as_bool('detailed') or False
+
+        try:
+            data = self._ctrl.get(flavor,
+                                  project=project_id,
+                                  detailed=detailed)
+
+        except errors.FlavorDoesNotExist as ex:
+            LOG.debug(ex)
+            raise falcon.HTTPNotFound()
+
+        data['href'] = request.path
+
+        # remove the name entry - it isn't needed on GET
+        del data['name']
+        response.body = transport_utils.to_json(data)
+        response.content_location = request.relative_uri
+
+    def on_put(self, request, response, project_id, flavor):
+        """Registers a new flavor. Expects the following input:
+
+        {"pool": "my-pool", "capabilities": {}}
+
+        A capabilities object may also be provided.
+
+        :returns: HTTP | [201]
+        """
+
+        # TODO(flaper87): Verify pool exists.
+
+        LOG.debug(u'PUT flavor - name: %s', flavor)
+
+        data = wsgi_utils.load(request)
+        wsgi_utils.validate(self._validators['create'], data)
+        self._ctrl.create(flavor,
+                          pool=data['pool'],
+                          project=project_id,
+                          capabilities=data['capabilities'])
+        response.status = falcon.HTTP_201
+        response.location = request.path
+
+    def on_delete(self, request, response, project_id, flavor):
+        """Deregisters a flavor.
+
+        :returns: HTTP | [204]
+
+        """
+
+        LOG.debug(u'DELETE flavor - name: %s', flavor)
+        self._ctrl.delete(flavor, project=project_id)
+        response.status = falcon.HTTP_204
+
+    def on_patch(self, request, response, project_id, flavor):
+        """Allows one to update a flavors's pool and/or capabilities.
+
+        This method expects the user to submit a JSON object
+        containing at least one of: 'pool', 'capabilities'. If
+        none are found, the request is flagged as bad. There is also
+        strict format checking through the use of
+        jsonschema. Appropriate errors are returned in each case for
+        badly formatted input.
+
+        :returns: HTTP | [200, 400]
+
+        """
+
+        LOG.debug(u'PATCH flavor - name: %s', flavor)
+        data = wsgi_utils.load(request)
+
+        EXPECT = ('pool', 'capabilities')
+        if not any([(field in data) for field in EXPECT]):
+            LOG.debug(u'PATCH flavor, bad params')
+            raise wsgi_errors.HTTPBadRequestBody(
+                'One of `pool` or `capabilities` needs '
+                'to be specified'
+            )
+
+        for field in EXPECT:
+            wsgi_utils.validate(self._validators[field], data)
+
+        fields = common_utils.fields(data, EXPECT,
+                                     pred=lambda v: v is not None)
+
+        try:
+            self._ctrl.update(flavor, project=project_id, **fields)
+        except errors.FlavorDoesNotExist as ex:
+            LOG.exception(ex)
+            raise falcon.HTTPNotFound()

zaqar/tests/queues/transport/wsgi/v1_1/__init__.py

@@ -15,6 +15,7 @@
 from zaqar.tests.queues.transport.wsgi.v1_1 import test_auth
 from zaqar.tests.queues.transport.wsgi.v1_1 import test_claims
 from zaqar.tests.queues.transport.wsgi.v1_1 import test_default_limits
+from zaqar.tests.queues.transport.wsgi.v1_1 import test_flavors
 from zaqar.tests.queues.transport.wsgi.v1_1 import test_health
 from zaqar.tests.queues.transport.wsgi.v1_1 import test_home
 from zaqar.tests.queues.transport.wsgi.v1_1 import test_media_type
@@ -42,3 +43,4 @@ TestQueueLifecycleSqlalchemy = l.TestQueueLifecycleSqlalchemy
 TestPoolsMongoDB = test_pools.TestPoolsMongoDB
 TestPoolsSqlalchemy = test_pools.TestPoolsSqlalchemy
 TestValidation = test_validation.TestValidation
+TestFlavorsMongoDB = test_flavors.TestFlavorsMongoDB

zaqar/tests/queues/transport/wsgi/v1_1/test_flavors.py

@@ -0,0 +1,275 @@
+# Copyright (c) 2014 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.  You may obtain a copy
+# of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import contextlib
+import uuid
+
+import ddt
+import falcon
+
+from zaqar.openstack.common import jsonutils
+from zaqar import tests as testing
+from zaqar.tests.queues.transport.wsgi import base
+
+
+@contextlib.contextmanager
+def flavor(test, name, pool, capabilities={}):
+    """A context manager for constructing a flavor for use in testing.
+
+    Deletes the flavor after exiting the context.
+
+    :param test: Must expose simulate_* methods
+    :param name: Name for this flavor
+    :type name: six.text_type
+    :type pool: six.text_type
+    :type capabilities: dict
+    :returns: (name, uri, capabilities)
+    :rtype: see above
+
+    """
+
+    doc = {'pool': pool, 'capabilities': capabilities}
+    path = test.url_prefix + '/flavors/' + name
+
+    test.simulate_put(path, body=jsonutils.dumps(doc))
+
+    try:
+        yield name, pool, capabilities
+
+    finally:
+        test.simulate_delete(path)
+
+
+@contextlib.contextmanager
+def flavors(test, count, pool):
+    """A context manager for constructing flavors for use in testing.
+
+    Deletes the flavors after exiting the context.
+
+    :param test: Must expose simulate_* methods
+    :param count: Number of pools to create
+    :type count: int
+    :returns: (paths, pool, capabilities)
+    :rtype: ([six.text_type], [six.text_type], [dict])
+
+    """
+
+    base = test.url_prefix + '/flavors/'
+    args = sorted([(base + str(i), {str(i): i}) for i in range(count)],
+                  key=lambda tup: tup[1])
+    for path, capabilities in args:
+        doc = {'pool': pool, 'capabilities': capabilities}
+        test.simulate_put(path, body=jsonutils.dumps(doc))
+
+    try:
+        yield args
+    finally:
+        for path, _ in args:
+            test.simulate_delete(path)
+
+
+@ddt.ddt
+class FlavorsBaseTest(base.V1_1Base):
+
+    def setUp(self):
+        super(FlavorsBaseTest, self).setUp()
+        self.flavor = 'test-flavor'
+        self.doc = {'capabilities': {}, 'pool': 'mypool'}
+        self.flavor = self.url_prefix + '/flavors/' + self.flavor
+        self.simulate_put(self.flavor, body=jsonutils.dumps(self.doc))
+        self.assertEqual(self.srmock.status, falcon.HTTP_201)
+
+    def tearDown(self):
+        super(FlavorsBaseTest, self).tearDown()
+        self.simulate_delete(self.flavor)
+        self.assertEqual(self.srmock.status, falcon.HTTP_204)
+
+    def test_put_flavor_works(self):
+        name = str(uuid.uuid1())
+        with flavor(self, name, self.doc['pool']):
+            self.assertEqual(self.srmock.status, falcon.HTTP_201)
+
+    def test_put_raises_if_missing_fields(self):
+        path = self.url_prefix + '/flavors/' + str(uuid.uuid1())
+        self.simulate_put(path, body=jsonutils.dumps({}))
+        self.assertEqual(self.srmock.status, falcon.HTTP_400)
+
+        self.simulate_put(path,
+                          body=jsonutils.dumps({'capabilities': {}}))
+        self.assertEqual(self.srmock.status, falcon.HTTP_400)
+
+    @ddt.data(1, 2**32+1, [])
+    def test_put_raises_if_invalid_pool(self, pool):
+        path = self.url_prefix + '/flavors/' + str(uuid.uuid1())
+        self.simulate_put(path,
+                          body=jsonutils.dumps({'pool': pool}))
+        self.assertEqual(self.srmock.status, falcon.HTTP_400)
+
+    @ddt.data(-1, 'wee', [])
+    def test_put_raises_if_invalid_capabilities(self, capabilities):
+        path = self.url_prefix + '/flavors/' + str(uuid.uuid1())
+        doc = {'pool': 'a', 'capabilities': capabilities}
+        self.simulate_put(path, body=jsonutils.dumps(doc))
+        self.assertEqual(self.srmock.status, falcon.HTTP_400)
+
+    def test_put_existing_overwrites(self):
+        # NOTE(cabrera): setUp creates default flavor
+        expect = self.doc
+        self.simulate_put(self.flavor,
+                          body=jsonutils.dumps(expect))
+        self.assertEqual(self.srmock.status, falcon.HTTP_201)
+
+        result = self.simulate_get(self.flavor)
+        self.assertEqual(self.srmock.status, falcon.HTTP_200)
+        doc = jsonutils.loads(result[0])
+        self.assertEqual(doc['pool'], expect['pool'])
+
+    def test_delete_works(self):
+        self.simulate_delete(self.flavor)
+        self.assertEqual(self.srmock.status, falcon.HTTP_204)
+
+        self.simulate_get(self.flavor)
+        self.assertEqual(self.srmock.status, falcon.HTTP_404)
+
+    def test_get_nonexisting_raises_404(self):
+        self.simulate_get(self.url_prefix + '/flavors/nonexisting')
+        self.assertEqual(self.srmock.status, falcon.HTTP_404)
+
+    def _flavor_expect(self, flavor, xhref, xpool):
+        self.assertIn('href', flavor)
+        self.assertEqual(flavor['href'], xhref)
+        self.assertIn('pool', flavor)
+        self.assertEqual(flavor['pool'], xpool)
+
+    def test_get_works(self):
+        result = self.simulate_get(self.flavor)
+        self.assertEqual(self.srmock.status, falcon.HTTP_200)
+        pool = jsonutils.loads(result[0])
+        self._flavor_expect(pool, self.flavor, self.doc['pool'])
+
+    def test_detailed_get_works(self):
+        result = self.simulate_get(self.flavor,
+                                   query_string='?detailed=True')
+        self.assertEqual(self.srmock.status, falcon.HTTP_200)
+        pool = jsonutils.loads(result[0])
+        self._flavor_expect(pool, self.flavor, self.doc['pool'])
+        self.assertIn('capabilities', pool)
+        self.assertEqual(pool['capabilities'], {})
+
+    def test_patch_raises_if_missing_fields(self):
+        self.simulate_patch(self.flavor,
+                            body=jsonutils.dumps({'location': 1}))
+        self.assertEqual(self.srmock.status, falcon.HTTP_400)
+
+    def _patch_test(self, doc):
+        self.simulate_patch(self.flavor,
+                            body=jsonutils.dumps(doc))
+        self.assertEqual(self.srmock.status, falcon.HTTP_200)
+
+        result = self.simulate_get(self.flavor,
+                                   query_string='?detailed=True')
+        self.assertEqual(self.srmock.status, falcon.HTTP_200)
+        pool = jsonutils.loads(result[0])
+        self._flavor_expect(pool, self.flavor, doc['pool'])
+        self.assertEqual(pool['capabilities'], doc['capabilities'])
+
+    def test_patch_works(self):
+        doc = {'pool': 'my-pool', 'capabilities': {'a': 1}}
+        self._patch_test(doc)
+
+    def test_patch_works_with_extra_fields(self):
+        doc = {'pool': 'my-pool', 'capabilities': {'a': 1},
+               'location': 100, 'partition': 'taco'}
+        self._patch_test(doc)
+
+    @ddt.data(-1, 2**32+1, [])
+    def test_patch_raises_400_on_invalid_pool(self, pool):
+        self.simulate_patch(self.flavor,
+                            body=jsonutils.dumps({'pool': pool}))
+        self.assertEqual(self.srmock.status, falcon.HTTP_400)
+
+    @ddt.data(-1, 'wee', [])
+    def test_patch_raises_400_on_invalid_capabilities(self, capabilities):
+        doc = {'capabilities': capabilities}
+        self.simulate_patch(self.flavor, body=jsonutils.dumps(doc))
+        self.assertEqual(self.srmock.status, falcon.HTTP_400)
+
+    def test_patch_raises_404_if_flavor_not_found(self):
+        self.simulate_patch(self.url_prefix + '/flavors/notexists',
+                            body=jsonutils.dumps({'pool': 'test'}))
+        self.assertEqual(self.srmock.status, falcon.HTTP_404)
+
+    def test_empty_listing_returns_204(self):
+        self.simulate_delete(self.flavor)
+        self.simulate_get(self.url_prefix + '/flavors')
+        self.assertEqual(self.srmock.status, falcon.HTTP_204)
+
+    def _listing_test(self, count=10, limit=10,
+                      marker=None, detailed=False):
+        # NOTE(cpp-cabrera): delete initial flavor - it will interfere
+        # with listing tests
+        self.simulate_delete(self.flavor)
+        query = '?limit={0}&detailed={1}'.format(limit, detailed)
+        if marker:
+            query += '&marker={2}'.format(marker)
+
+        with flavors(self, count, self.doc['pool']) as expected:
+            result = self.simulate_get(self.url_prefix + '/flavors',
+                                       query_string=query)
+            self.assertEqual(self.srmock.status, falcon.HTTP_200)
+            results = jsonutils.loads(result[0])
+            self.assertIsInstance(results, dict)
+            self.assertIn('flavors', results)
+            flavors_list = results['flavors']
+            self.assertEqual(len(flavors_list), min(limit, count))
+            for i, s in enumerate(flavors_list):
+                expect = expected[i]
+                path, capabilities = expect[:2]
+                self._flavor_expect(s, path, self.doc['pool'])
+                if detailed:
+                    self.assertIn('capabilities', s)
+                    self.assertEqual(s['capabilities'], capabilities)
+                else:
+                    self.assertNotIn('capabilities', s)
+
+    def test_listing_works(self):
+        self._listing_test()
+
+    def test_detailed_listing_works(self):
+        self._listing_test(detailed=True)
+
+    @ddt.data(1, 5, 10, 15)
+    def test_listing_works_with_limit(self, limit):
+        self._listing_test(count=15, limit=limit)
+
+    def test_listing_marker_is_respected(self):
+        self.simulate_delete(self.flavor)
+
+        with flavors(self, 10, self.doc['pool']) as expected:
+            result = self.simulate_get(self.url_prefix + '/flavors',
+                                       query_string='?marker=3')
+            self.assertEqual(self.srmock.status, falcon.HTTP_200)
+            flavor_list = jsonutils.loads(result[0])['flavors']
+            self.assertEqual(len(flavor_list), 6)
+            path, capabilities = expected[4][:2]
+            self._flavor_expect(flavor_list[0], path, self.doc['pool'])
+
+
+class TestFlavorsMongoDB(FlavorsBaseTest):
+
+    config_file = 'wsgi_mongodb.conf'
+
+    @testing.requires_mongodb
+    def setUp(self):
+        super(TestFlavorsMongoDB, self).setUp()