From 9e5754695df34587ca6545a67c48f8c78584978d Mon Sep 17 00:00:00 2001 From: Zhihao Yuan Date: Tue, 20 Aug 2013 11:44:08 -0400 Subject: [PATCH] feat(storage): configurable default paging size This change add the following options to the config file: [limits:storage] default_queue_paging = 10 default_message_paging = 10 So that the default value of the "limit" URI param is now configurable. This patch also removes the "actions" cruft. Implements: blueprint configurable-default-paging Change-Id: Id38295f1e607226a4259be7744e6ce2d7b6de12e --- etc/marconi.conf-sample | 6 ++ marconi/storage/base.py | 19 +--- marconi/storage/mongodb/claims.py | 10 ++- marconi/storage/mongodb/messages.py | 9 +- marconi/storage/mongodb/queues.py | 14 +-- marconi/storage/sqlite/claims.py | 11 ++- marconi/storage/sqlite/messages.py | 10 ++- marconi/storage/sqlite/queues.py | 13 ++- .../tests/etc/wsgi_sqlite_default_limits.conf | 7 ++ .../transport/wsgi/test_default_limits.py | 86 +++++++++++++++++++ marconi/tests/util/faulty_storage.py | 3 - 11 files changed, 156 insertions(+), 32 deletions(-) create mode 100644 marconi/tests/etc/wsgi_sqlite_default_limits.conf create mode 100644 marconi/tests/transport/wsgi/test_default_limits.py diff --git a/etc/marconi.conf-sample b/etc/marconi.conf-sample index 86ca72dac..48557ffbc 100644 --- a/etc/marconi.conf-sample +++ b/etc/marconi.conf-sample @@ -84,3 +84,9 @@ database = marconi # for each metadata body and each message body ;metadata_size_uplimit = 65536 ;message_size_uplimit = 262144 + +[limits:storage] +# The default number of queue records per page when listing queues +;default_queue_paging = 10 +# The default number of messages per page when listing or claiming messages +;default_message_paging = 10 diff --git a/marconi/storage/base.py b/marconi/storage/base.py index 99145e2b9..2a05bc14b 100644 --- a/marconi/storage/base.py +++ b/marconi/storage/base.py @@ -90,7 +90,8 @@ class QueueBase(ControllerBase): :param project: Project id :param marker: The last queue name - :param limit: (Default 10) Max number + :param limit: (Default 10, configurable) Max number + queues to return. :param detailed: Whether metadata is included :param include_claimed: Whether to list claimed messages @@ -164,18 +165,6 @@ class QueueBase(ControllerBase): """ raise NotImplementedError - @abc.abstractmethod - def actions(self, name, project=None, marker=None, limit=10): - """Base method for queue actions. - - :param name: Queue name - :param project: Project id - :param marker: Tail identifier - :param limit: (Default 10) Max number - of messages to retrieve. - """ - raise NotImplementedError - class MessageBase(ControllerBase): """This class is responsible for managing message CRUD.""" @@ -191,7 +180,7 @@ class MessageBase(ControllerBase): message from. :param project: Project id :param marker: Tail identifier - :param limit: (Default 10) specifies up to 100 + :param limit: (Default 10, configurable) Max number messages to return. :param echo: (Default False) Boolean expressing whether or not this client should receive its own messages. @@ -302,7 +291,7 @@ class ClaimBase(ControllerBase): :param metadata: Claim's parameters to be stored. :param project: Project id - :param limit: (Default 10) Max number + :param limit: (Default 10, configurable) Max number of messages to claim. :returns: (Claim ID, claimed messages) diff --git a/marconi/storage/mongodb/claims.py b/marconi/storage/mongodb/claims.py index d24c56d76..8a0562a3a 100644 --- a/marconi/storage/mongodb/claims.py +++ b/marconi/storage/mongodb/claims.py @@ -25,14 +25,17 @@ import datetime from bson import objectid +from marconi.common import config import marconi.openstack.common.log as logging from marconi.openstack.common import timeutils from marconi import storage from marconi.storage import exceptions from marconi.storage.mongodb import utils - LOG = logging.getLogger(__name__) +CFG = config.namespace('limits:storage').from_options( + default_message_paging=10, +) class ClaimController(storage.ClaimBase): @@ -98,7 +101,7 @@ class ClaimController(storage.ClaimBase): return (claim, msgs) @utils.raises_conn_error - def create(self, queue, metadata, project=None, limit=10): + def create(self, queue, metadata, project=None, limit=None): """Creates a claim. This implementation was done in a best-effort fashion. @@ -118,6 +121,9 @@ class ClaimController(storage.ClaimBase): """ msg_ctrl = self.driver.message_controller + if limit is None: + limit = CFG.default_message_paging + ttl = metadata['ttl'] grace = metadata['grace'] oid = objectid.ObjectId() diff --git a/marconi/storage/mongodb/messages.py b/marconi/storage/mongodb/messages.py index b85dfb82c..d39f79d20 100644 --- a/marconi/storage/mongodb/messages.py +++ b/marconi/storage/mongodb/messages.py @@ -26,6 +26,7 @@ import time import pymongo.errors +from marconi.common import config import marconi.openstack.common.log as logging from marconi.openstack.common import timeutils from marconi import storage @@ -34,6 +35,9 @@ from marconi.storage.mongodb import options from marconi.storage.mongodb import utils LOG = logging.getLogger(__name__) +CFG = config.namespace('limits:storage').from_options( + default_message_paging=10, +) class MessageController(storage.MessageBase): @@ -397,9 +401,12 @@ class MessageController(storage.MessageBase): for name, project in self._queue_controller._get_np(): self._remove_expired(name, project) - def list(self, queue_name, project=None, marker=None, limit=10, + def list(self, queue_name, project=None, marker=None, limit=None, echo=False, client_uuid=None, include_claimed=False): + if limit is None: + limit = CFG.default_message_paging + if marker is not None: try: marker = int(marker) diff --git a/marconi/storage/mongodb/queues.py b/marconi/storage/mongodb/queues.py index 5d2fc534c..f5c5500dc 100644 --- a/marconi/storage/mongodb/queues.py +++ b/marconi/storage/mongodb/queues.py @@ -23,6 +23,7 @@ Field Mappings: import pymongo.errors +from marconi.common import config import marconi.openstack.common.log as logging from marconi.openstack.common import timeutils from marconi import storage @@ -30,6 +31,9 @@ from marconi.storage import exceptions from marconi.storage.mongodb import utils LOG = logging.getLogger(__name__) +CFG = config.namespace('limits:storage').from_options( + default_queue_paging=10, +) class QueueController(storage.QueueBase): @@ -77,7 +81,11 @@ class QueueController(storage.QueueBase): #----------------------------------------------------------------------- def list(self, project=None, marker=None, - limit=10, detailed=False): + limit=None, detailed=False): + + if limit is None: + limit = CFG.default_queue_paging + query = {'p': project} if marker: query['n'] = {'$gt': marker} @@ -161,7 +169,3 @@ class QueueController(storage.QueueBase): message_stats['newest'] = utils.stat_message(newest, now) return {'messages': message_stats} - - @utils.raises_conn_error - def actions(self, name, project=None, marker=None, limit=10): - raise NotImplementedError diff --git a/marconi/storage/sqlite/claims.py b/marconi/storage/sqlite/claims.py index d46358fb4..92aa7c8d5 100644 --- a/marconi/storage/sqlite/claims.py +++ b/marconi/storage/sqlite/claims.py @@ -13,10 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +from marconi.common import config from marconi.storage import base from marconi.storage import exceptions from marconi.storage.sqlite import utils +CFG = config.namespace('limits:storage').from_options( + default_message_paging=10, +) + class ClaimController(base.ClaimBase): def __init__(self, driver): @@ -73,10 +78,14 @@ class ClaimController(base.ClaimBase): except utils.NoResult: raise exceptions.ClaimDoesNotExist(claim_id, queue, project) - def create(self, queue, metadata, project, limit=10): + def create(self, queue, metadata, project, limit=None): + if project is None: project = '' + if limit is None: + limit = CFG.default_message_paging + with self.driver('immediate'): try: qid = utils.get_qid(self.driver, queue, project) diff --git a/marconi/storage/sqlite/messages.py b/marconi/storage/sqlite/messages.py index fc29612cf..600b0143d 100644 --- a/marconi/storage/sqlite/messages.py +++ b/marconi/storage/sqlite/messages.py @@ -13,11 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +from marconi.common import config from marconi.openstack.common import timeutils from marconi.storage import base from marconi.storage import exceptions from marconi.storage.sqlite import utils +CFG = config.namespace('limits:storage').from_options( + default_message_paging=10, +) + class MessageController(base.MessageBase): def __init__(self, driver): @@ -130,9 +135,12 @@ class MessageController(base.MessageBase): 'body': content, } - def list(self, queue, project, marker=None, limit=10, + def list(self, queue, project, marker=None, limit=None, echo=False, client_uuid=None, include_claimed=False): + if limit is None: + limit = CFG.default_message_paging + if project is None: project = '' diff --git a/marconi/storage/sqlite/queues.py b/marconi/storage/sqlite/queues.py index f80d5268a..e4fc7dd32 100644 --- a/marconi/storage/sqlite/queues.py +++ b/marconi/storage/sqlite/queues.py @@ -14,10 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +from marconi.common import config from marconi.storage import base from marconi.storage import exceptions from marconi.storage.sqlite import utils +CFG = config.namespace('limits:storage').from_options( + default_queue_paging=10, +) + class QueueController(base.QueueBase): def __init__(self, driver): @@ -36,11 +41,14 @@ class QueueController(base.QueueBase): ''') def list(self, project, marker=None, - limit=10, detailed=False): + limit=None, detailed=False): if project is None: project = '' + if limit is None: + limit = CFG.default_queue_paging + sql = ((''' select name from Queues''' if not detailed else ''' @@ -166,6 +174,3 @@ class QueueController(base.QueueBase): message_stats['newest'] = utils.stat_message(newest) return {'messages': message_stats} - - def actions(self, name, project, marker=None, limit=10): - raise NotImplementedError diff --git a/marconi/tests/etc/wsgi_sqlite_default_limits.conf b/marconi/tests/etc/wsgi_sqlite_default_limits.conf new file mode 100644 index 000000000..b0e6d65a2 --- /dev/null +++ b/marconi/tests/etc/wsgi_sqlite_default_limits.conf @@ -0,0 +1,7 @@ +[drivers] +transport = wsgi +storage = sqlite + +[limits:storage] +default_queue_paging = 1 +default_message_paging = 2 diff --git a/marconi/tests/transport/wsgi/test_default_limits.py b/marconi/tests/transport/wsgi/test_default_limits.py new file mode 100644 index 000000000..55a12d9ec --- /dev/null +++ b/marconi/tests/transport/wsgi/test_default_limits.py @@ -0,0 +1,86 @@ +# Copyright (c) 2013 Rackspace, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import falcon + +from marconi.tests.transport.wsgi import base + + +class DefaultLimitsTest(base.TestBase): + + config_filename = 'wsgi_sqlite_default_limits.conf' + + def setUp(self): + super(DefaultLimitsTest, self).setUp() + + self.queue_path = '/v1/queues/q1' + self.messages_path = self.queue_path + '/messages' + self.claims_path = self.queue_path + '/claims' + + self.simulate_put(self.queue_path) + + def tearDown(self): + self.simulate_delete(self.queue_path) + super(DefaultLimitsTest, self).tearDown() + + def test_queue_listing(self): + default_queue_paging = 1 + + # 2 queues to list + self.simulate_put('/v1/queues/q2') + self.assertEquals(self.srmock.status, falcon.HTTP_201) + + result = self.simulate_get('/v1/queues') + self.assertEquals(self.srmock.status, falcon.HTTP_200) + + queues = json.loads(result[0])['queues'] + self.assertEquals(len(queues), default_queue_paging) + + self.simulate_delete('/v1/queues/q2') + + def test_message_listing(self): + default_message_paging = 2 + + # 10 messages to list + self.__prepare_messages(10) + + result = self.simulate_get(self.messages_path, + headers={'Client-ID': 'audience'}) + + self.assertEquals(self.srmock.status, falcon.HTTP_200) + + messages = json.loads(result[0])['messages'] + self.assertEquals(len(messages), default_message_paging) + + def test_claim_creation(self): + default_message_paging = 2 + + # 5 messages to claim + self.__prepare_messages(5) + + result = self.simulate_post(self.claims_path, + body='{"ttl": 60, "grace": 60}') + + self.assertEquals(self.srmock.status, falcon.HTTP_201) + + messages = json.loads(result[0]) + self.assertEquals(len(messages), default_message_paging) + + def __prepare_messages(self, count): + doc = json.dumps([{'body': 239, 'ttl': 300}] * count) + self.simulate_post(self.messages_path, body=doc, + headers={'Client-ID': 'poster'}) diff --git a/marconi/tests/util/faulty_storage.py b/marconi/tests/util/faulty_storage.py index 8575e6d8c..f3724f3be 100644 --- a/marconi/tests/util/faulty_storage.py +++ b/marconi/tests/util/faulty_storage.py @@ -56,9 +56,6 @@ class QueueController(storage.QueueBase): def stats(self, name, project=None): raise NotImplementedError() - def actions(self, name, project=None, marker=None, limit=10): - raise NotImplementedError() - class MessageController(storage.MessageBase): def __init__(self, driver):