From 7aa2522e3d370a70882d07d7641741679616fa55 Mon Sep 17 00:00:00 2001 From: wanghao Date: Thu, 9 Jan 2020 12:09:31 +0800 Subject: [PATCH] Support query queues with count Zaqar will support query queues with 'with_count' to return the amount of the queues. This will help users to quickly get the exact total number of queues which they own. Change-Id: I1d2cdc802ecd76c01671cd5660ae79dd39505d43 Implements: blueprint query-queues-with-count Signed-off-by: wanghao --- api-ref/source/parameters.yaml | 15 +++++++++++++++ api-ref/source/queues.inc | 2 ++ .../source/samples/queues-list-response.json | 3 ++- ...ery-quques-with-count-4453825671bb5298.yaml | 5 +++++ zaqar/storage/base.py | 10 ++++++++++ zaqar/storage/mongodb/queues.py | 7 +++++++ zaqar/storage/pooling.py | 3 +++ zaqar/storage/redis/queues.py | 9 ++++++++- zaqar/storage/sqlalchemy/queues.py | 10 ++++++++++ zaqar/tests/faulty_storage.py | 3 +++ .../wsgi/v2_0/test_queue_lifecycle.py | 7 +++++++ zaqar/transport/wsgi/v2_0/queues.py | 18 ++++++++++++++---- 12 files changed, 86 insertions(+), 6 deletions(-) create mode 100644 releasenotes/notes/support-query-quques-with-count-4453825671bb5298.yaml diff --git a/api-ref/source/parameters.yaml b/api-ref/source/parameters.yaml index 4eab64d3a..b2c788ea8 100644 --- a/api-ref/source/parameters.yaml +++ b/api-ref/source/parameters.yaml @@ -151,6 +151,14 @@ pop: ``pop`` & ``ids`` parameters are mutually exclusive. Using them together in a request will result in HTTP 400. +with_count: + type: boolean + in: query + required: false + description: | + The 'with_count' specifies if showing the amount of queues when querying + them. + # variables in body _dead_letter_queue: @@ -312,6 +320,13 @@ claim_ttl: the claim. The ttl value must be between 60 and 43200 seconds (12 hours). You must include a value for this attribute in your request. +count: + type: integer + in: body + required: false + description: | + The ``count`` attribute specifies how many queus in current project. + flavor_href: type: string in: body diff --git a/api-ref/source/queues.inc b/api-ref/source/queues.inc index ee530f99c..49cedcac1 100644 --- a/api-ref/source/queues.inc +++ b/api-ref/source/queues.inc @@ -50,6 +50,7 @@ Request Parameters - marker: marker - detailed: detailed - name: name + - with_count: with_count Response Parameters ------------------- @@ -58,6 +59,7 @@ Response Parameters - queues: queues - links: links + - count: count Response Example diff --git a/api-ref/source/samples/queues-list-response.json b/api-ref/source/samples/queues-list-response.json index 05b79bbf8..2f4fe6cde 100644 --- a/api-ref/source/samples/queues-list-response.json +++ b/api-ref/source/samples/queues-list-response.json @@ -18,5 +18,6 @@ "href":"/v2/queues?marker=wellington", "rel":"next" } - ] + ], + "count": 3 } \ No newline at end of file diff --git a/releasenotes/notes/support-query-quques-with-count-4453825671bb5298.yaml b/releasenotes/notes/support-query-quques-with-count-4453825671bb5298.yaml new file mode 100644 index 000000000..526ace535 --- /dev/null +++ b/releasenotes/notes/support-query-quques-with-count-4453825671bb5298.yaml @@ -0,0 +1,5 @@ +--- +features: + - Support query queues with filter 'with_count=true' to return the amount of + the queues. This will help users to quickly get the exact total number of + queues which they own. diff --git a/zaqar/storage/base.py b/zaqar/storage/base.py index 80335c474..2361e7c2b 100644 --- a/zaqar/storage/base.py +++ b/zaqar/storage/base.py @@ -418,6 +418,16 @@ class Queue(ControllerBase): _stats = abc.abstractmethod(lambda x: None) + def calculate_resource_count(self, project=None): + """Base method for calculate queues amount. + + :param project: Project id + :returns: The number of queues. + """ + return self._calculate_resource_count(project) + + _calculate_resource_count = abc.abstractmethod(lambda x: None) + @six.add_metaclass(abc.ABCMeta) class Message(ControllerBase): diff --git a/zaqar/storage/mongodb/queues.py b/zaqar/storage/mongodb/queues.py index 9938896d6..bda228ac1 100644 --- a/zaqar/storage/mongodb/queues.py +++ b/zaqar/storage/mongodb/queues.py @@ -287,6 +287,13 @@ class QueueController(storage.Queue): def _stats(self, name, project=None): pass + @utils.raises_conn_error + @utils.retries_on_autoreconnect + def _calculate_resource_count(self, project=None): + query = utils.scoped_query(None, project, None, {}) + projection = {'p_q': 1, '_id': 0} + return self._collection.find(query, projection=projection).count() + def _get_scoped_query(name, project): return {'p_q': utils.scope_queue_name(name, project)} diff --git a/zaqar/storage/pooling.py b/zaqar/storage/pooling.py index fae42eb60..aace70ab3 100644 --- a/zaqar/storage/pooling.py +++ b/zaqar/storage/pooling.py @@ -264,6 +264,9 @@ class QueueController(storage.Queue): return mqHandler.stats(name, project=project) raise errors.QueueDoesNotExist(name, project) + def _calculate_resource_count(self, project=None): + return self._mgt_queue_ctrl.calculate_resource_count(project=project) + class MessageController(storage.Message): """Routes operations to a message controller in the appropriate pool. diff --git a/zaqar/storage/redis/queues.py b/zaqar/storage/redis/queues.py index fa756804b..67fbdda4b 100644 --- a/zaqar/storage/redis/queues.py +++ b/zaqar/storage/redis/queues.py @@ -103,7 +103,7 @@ class QueueController(storage.Queue): queue = {'name': utils.descope_queue_name(name)} marker_next['next'] = queue['name'] if detailed: - queue['metadata'] = info[1] + queue['metadata'] = self._unpacker(info[1]) return queue @@ -192,3 +192,10 @@ class QueueController(storage.Queue): @utils.retries_on_connection_error def _stats(self, name, project=None): pass + + @utils.raises_conn_error + @utils.retries_on_connection_error + def _calculate_resource_count(self, project=None): + client = self._client + qset_key = utils.scope_queue_name(QUEUES_SET_STORE_NAME, project) + return client.zlexcount(qset_key, '-', '+') diff --git a/zaqar/storage/sqlalchemy/queues.py b/zaqar/storage/sqlalchemy/queues.py index 5ebd0def4..1fda8c40f 100644 --- a/zaqar/storage/sqlalchemy/queues.py +++ b/zaqar/storage/sqlalchemy/queues.py @@ -134,3 +134,13 @@ class QueueController(storage.Queue): def _stats(self, name, project): pass + + def _calculate_resource_count(self, project=None): + if project is None: + project = '' + sel = sa.sql.select([sa.sql.func.count('*')], + tables.Queues.c.project == project) + res = self.driver.run(sel) + r = res.fetchone() + res.close() + return r is not None diff --git a/zaqar/tests/faulty_storage.py b/zaqar/tests/faulty_storage.py index 2be44faf4..f87e4fb6f 100644 --- a/zaqar/tests/faulty_storage.py +++ b/zaqar/tests/faulty_storage.py @@ -123,6 +123,9 @@ class QueueController(storage.Queue): def _stats(self, name, project=None): raise NotImplementedError() + def _calculate_resource_count(self, project=None): + raise NotImplementedError() + class MessageController(storage.Message): def __init__(self, driver): diff --git a/zaqar/tests/unit/transport/wsgi/v2_0/test_queue_lifecycle.py b/zaqar/tests/unit/transport/wsgi/v2_0/test_queue_lifecycle.py index a040cf005..633895645 100644 --- a/zaqar/tests/unit/transport/wsgi/v2_0/test_queue_lifecycle.py +++ b/zaqar/tests/unit/transport/wsgi/v2_0/test_queue_lifecycle.py @@ -587,6 +587,13 @@ class TestQueueLifecycleMongoDB(base.V2Base): result_doc = jsonutils.loads(result[0]) self.assertEqual(3, len(result_doc['queues'])) + # List (filter query) + result = self.simulate_get(self.queue_path, headers=header, + query_string='with_count=true') + + result_doc = jsonutils.loads(result[0]) + self.assertEqual(3, result_doc['count']) + class TestQueueLifecycleFaultyDriver(base.V2BaseFaulty): diff --git a/zaqar/transport/wsgi/v2_0/queues.py b/zaqar/transport/wsgi/v2_0/queues.py index 59f322cdb..faf014a5f 100644 --- a/zaqar/transport/wsgi/v2_0/queues.py +++ b/zaqar/transport/wsgi/v2_0/queues.py @@ -256,12 +256,17 @@ class CollectionResource(object): def _queue_list(self, project_id, path, kfilter, **kwargs): try: self._validate.queue_listing(**kwargs) + with_count = kwargs.pop('with_count', False) results = self._queue_controller.list(project=project_id, kfilter=kfilter, **kwargs) # Buffer list of queues queues = list(next(results)) + total_number = None + if with_count: + total_number = self._queue_controller.calculate_resource_count( + project=project_id) except validation.ValidationFailed as ex: LOG.debug(ex) raise wsgi_errors.HTTPBadRequestAPI(six.text_type(ex)) @@ -281,7 +286,7 @@ class CollectionResource(object): if not each_queue.get('metadata', {}).get(meta): each_queue['metadata'][meta] = value - return queues, kwargs['marker'] + return queues, kwargs['marker'], total_number def _on_get_with_kfilter(self, req, resp, project_id, kfilter={}): kwargs = {} @@ -292,9 +297,11 @@ class CollectionResource(object): req.get_param_as_int('limit', store=kwargs) req.get_param_as_bool('detailed', store=kwargs) req.get_param('name', store=kwargs) + req.get_param_as_bool('with_count', store=kwargs) - queues, marker = self._queue_list(project_id, - req.path, kfilter, **kwargs) + queues, marker, total_number = self._queue_list(project_id, + req.path, kfilter, + **kwargs) links = [] kwargs['marker'] = marker @@ -311,13 +318,16 @@ class CollectionResource(object): 'links': links } + if total_number: + response_body['count'] = total_number + resp.body = utils.to_json(response_body) # status defaults to 200 @decorators.TransportLog("Queues collection") @acl.enforce("queues:get_all") def on_get(self, req, resp, project_id): - field = ('marker', 'limit', 'detailed', 'name') + field = ('marker', 'limit', 'detailed', 'name', 'with_count') kfilter = copy.deepcopy(req.params) for key in req.params.keys():