Support query queues with count

Zaqar will support query queues with 'with_count' to
return the amount of the queues. This will help users to
quickly get the exact total number of queues which they own.

Change-Id: I1d2cdc802ecd76c01671cd5660ae79dd39505d43
Implements: blueprint query-queues-with-count
Signed-off-by: wanghao <sxmatch1986@gmail.com>

api-ref/source/parameters.yaml

@@ -151,6 +151,14 @@ pop:
     ``pop`` & ``ids`` parameters are mutually exclusive. Using them together
     in a request will result in HTTP 400.
 
+with_count:
+  type: boolean
+  in: query
+  required: false
+  description: |
+    The 'with_count' specifies if showing the amount of queues when querying
+    them.
+
 # variables in body
 
 _dead_letter_queue:
@@ -312,6 +320,13 @@ claim_ttl:
     the claim. The ttl value must be between 60 and 43200 seconds (12 hours).
     You must include a value for this attribute in your request.
 
+count:
+  type: integer
+  in: body
+  required: false
+  description: |
+    The ``count`` attribute specifies how many queus in current project.
+
 flavor_href:
   type: string
   in: body

api-ref/source/queues.inc

@@ -50,6 +50,7 @@ Request Parameters
   - marker: marker
   - detailed: detailed
   - name: name
+  - with_count: with_count
 
 Response Parameters
 -------------------
@@ -58,6 +59,7 @@ Response Parameters
 
   - queues: queues
   - links: links
+  - count: count
 
 
 Response Example

api-ref/source/samples/queues-list-response.json

@@ -18,5 +18,6 @@
          "href":"/v2/queues?marker=wellington",
          "rel":"next"
       }
-   ]
+   ],
+   "count": 3
 }

releasenotes/notes/support-query-quques-with-count-4453825671bb5298.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - Support query queues with filter 'with_count=true' to return the amount of
+    the queues. This will help users to quickly get the exact total number of
+    queues which they own.

zaqar/storage/base.py

@@ -418,6 +418,16 @@ class Queue(ControllerBase):
 
     _stats = abc.abstractmethod(lambda x: None)
 
+    def calculate_resource_count(self, project=None):
+        """Base method for calculate queues amount.
+
+        :param project: Project id
+        :returns: The number of queues.
+        """
+        return self._calculate_resource_count(project)
+
+    _calculate_resource_count = abc.abstractmethod(lambda x: None)
+
 
 @six.add_metaclass(abc.ABCMeta)
 class Message(ControllerBase):

zaqar/storage/mongodb/queues.py

@@ -287,6 +287,13 @@ class QueueController(storage.Queue):
     def _stats(self, name, project=None):
         pass
 
+    @utils.raises_conn_error
+    @utils.retries_on_autoreconnect
+    def _calculate_resource_count(self, project=None):
+        query = utils.scoped_query(None, project, None, {})
+        projection = {'p_q': 1, '_id': 0}
+        return self._collection.find(query, projection=projection).count()
+
 
 def _get_scoped_query(name, project):
     return {'p_q': utils.scope_queue_name(name, project)}

zaqar/storage/pooling.py

@@ -264,6 +264,9 @@ class QueueController(storage.Queue):
             return mqHandler.stats(name, project=project)
         raise errors.QueueDoesNotExist(name, project)
 
+    def _calculate_resource_count(self, project=None):
+        return self._mgt_queue_ctrl.calculate_resource_count(project=project)
+
 
 class MessageController(storage.Message):
     """Routes operations to a message controller in the appropriate pool.

zaqar/storage/redis/queues.py

@@ -103,7 +103,7 @@ class QueueController(storage.Queue):
             queue = {'name': utils.descope_queue_name(name)}
             marker_next['next'] = queue['name']
             if detailed:
-                queue['metadata'] = info[1]
+                queue['metadata'] = self._unpacker(info[1])
 
             return queue
 
@@ -192,3 +192,10 @@ class QueueController(storage.Queue):
     @utils.retries_on_connection_error
     def _stats(self, name, project=None):
         pass
+
+    @utils.raises_conn_error
+    @utils.retries_on_connection_error
+    def _calculate_resource_count(self, project=None):
+        client = self._client
+        qset_key = utils.scope_queue_name(QUEUES_SET_STORE_NAME, project)
+        return client.zlexcount(qset_key, '-', '+')

zaqar/storage/sqlalchemy/queues.py

@@ -134,3 +134,13 @@ class QueueController(storage.Queue):
 
     def _stats(self, name, project):
         pass
+
+    def _calculate_resource_count(self, project=None):
+        if project is None:
+            project = ''
+        sel = sa.sql.select([sa.sql.func.count('*')],
+                            tables.Queues.c.project == project)
+        res = self.driver.run(sel)
+        r = res.fetchone()
+        res.close()
+        return r is not None

zaqar/tests/faulty_storage.py

@@ -123,6 +123,9 @@ class QueueController(storage.Queue):
     def _stats(self, name, project=None):
         raise NotImplementedError()
 
+    def _calculate_resource_count(self, project=None):
+        raise NotImplementedError()
+
 
 class MessageController(storage.Message):
     def __init__(self, driver):

zaqar/tests/unit/transport/wsgi/v2_0/test_queue_lifecycle.py

@@ -587,6 +587,13 @@ class TestQueueLifecycleMongoDB(base.V2Base):
         result_doc = jsonutils.loads(result[0])
         self.assertEqual(3, len(result_doc['queues']))
 
+        # List (filter query)
+        result = self.simulate_get(self.queue_path, headers=header,
+                                   query_string='with_count=true')
+
+        result_doc = jsonutils.loads(result[0])
+        self.assertEqual(3, result_doc['count'])
+
 
 class TestQueueLifecycleFaultyDriver(base.V2BaseFaulty):
 

zaqar/transport/wsgi/v2_0/queues.py

@@ -256,12 +256,17 @@ class CollectionResource(object):
     def _queue_list(self, project_id, path, kfilter, **kwargs):
         try:
             self._validate.queue_listing(**kwargs)
+            with_count = kwargs.pop('with_count', False)
             results = self._queue_controller.list(project=project_id,
                                                   kfilter=kfilter, **kwargs)
 
             # Buffer list of queues
             queues = list(next(results))
 
+            total_number = None
+            if with_count:
+                total_number = self._queue_controller.calculate_resource_count(
+                    project=project_id)
         except validation.ValidationFailed as ex:
             LOG.debug(ex)
             raise wsgi_errors.HTTPBadRequestAPI(six.text_type(ex))
@@ -281,7 +286,7 @@ class CollectionResource(object):
                     if not each_queue.get('metadata', {}).get(meta):
                         each_queue['metadata'][meta] = value
 
-        return queues, kwargs['marker']
+        return queues, kwargs['marker'], total_number
 
     def _on_get_with_kfilter(self, req, resp, project_id, kfilter={}):
         kwargs = {}
@@ -292,9 +297,11 @@ class CollectionResource(object):
         req.get_param_as_int('limit', store=kwargs)
         req.get_param_as_bool('detailed', store=kwargs)
         req.get_param('name', store=kwargs)
+        req.get_param_as_bool('with_count', store=kwargs)
 
-        queues, marker = self._queue_list(project_id,
-                                          req.path, kfilter, **kwargs)
+        queues, marker, total_number = self._queue_list(project_id,
+                                                        req.path, kfilter,
+                                                        **kwargs)
 
         links = []
         kwargs['marker'] = marker
@@ -311,13 +318,16 @@ class CollectionResource(object):
             'links': links
         }
 
+        if total_number:
+            response_body['count'] = total_number
+
         resp.body = utils.to_json(response_body)
         # status defaults to 200
 
     @decorators.TransportLog("Queues collection")
     @acl.enforce("queues:get_all")
     def on_get(self, req, resp, project_id):
-        field = ('marker', 'limit', 'detailed', 'name')
+        field = ('marker', 'limit', 'detailed', 'name', 'with_count')
         kfilter = copy.deepcopy(req.params)
 
         for key in req.params.keys():