Enhance Vitrage resource APIs

Add a new API for retrieving the number of resources,
 similar to 'vitrage alarm count'.
The API includes filter and group-by options.
Add filter option to 'vitrage resource list'

Story: 2004669
Task: 28650
Task: 28656

Change-Id: I3b9f2be581dcbd4a539b8040a387512b55597953

doc/source/contributor/vitrage-api.rst

@@ -1259,8 +1259,8 @@ Resource list
 ^^^^^^^^^^^^^
 List the resources with specified type or all the resources.
 
-GET /v1/resources/
+POST /v1/resources/
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~
 
 Headers
 =======
@@ -1278,20 +1278,21 @@ None.
 Query Parameters
 ================
 
-* resource_type - (string, optional) the type of resource, defaults to return all resources.
+None.
-* all_tenants - (boolean, optional) shows the resources of all tenants (in case the user has the permissions).
 
 Request Body
 ============
 
-None.
+* resource_type - (string, optional) the type of resource, defaults to return all resources.
+* all_tenants - (boolean, optional) shows the resources of all tenants (in case the user has the permissions).
+* query - (string, optional) a json query to filter the resources by
 
 Request Examples
 ================
 
 ::
 
-    GET /v1/resources/?all_tenants=False&resource_type=nova.host
+    POST /v1/resources/
     Host: 135.248.18.122:8999
     User-Agent: keystoneauth1/2.3.0 python-requests/2.9.1 CPython/2.7.6
     Content-Type: application/json
@@ -1305,6 +1306,37 @@ Response Status code
 -  200 - OK
 -  404 - Bad request
 
+Query example
+=============
+
+::
+
+    POST /v1/resources/
+    Host: 135.248.19.18:8999
+    Content-Type: application/json
+    X-Auth-Token: 2b8882ba2ec44295bf300aecb2caa4f7
+
+    {
+        "query" :"
+         {
+            \"or\": [
+              {
+                \"==\": {
+                  \"state\": \"OK\"
+                }
+              },
+              {
+                \"==\": {
+                  \"state\": \"SUBOPTIMAL\"
+                }
+              }
+            ]
+
+         }",
+         "resource_type" : "nova.host"
+         "all_tenants" : True
+     }
+
 Response Body
 =============
 
@@ -1400,6 +1432,108 @@ Response Examples
       "vitrage_id": "11680c27-86a2-41a7-89db-863e68b1c2c9"
     }
 
+Resource count
+^^^^^^^^^^^^^^
+Count resources
+
+POST /v1/resources/count
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Headers
+=======
+
+-  X-Auth-Token (string, required) - Keystone auth token
+-  Accept (string) - application/json
+-  User-Agent (String)
+-  Content-Type (String): application/json
+
+Path Parameters
+===============
+
+None.
+
+Query Parameters
+================
+
+None.
+
+Request Body
+============
+
+* resource_type - (string, optional) the type of resource, defaults to return all resources.
+* all_tenants - (boolean, optional) shows the resources of all tenants (in case the user has the permissions).
+* query - (string, optional) a json query to filter the resources by
+* group_by - (string, optional) a resource data field, to group by its values
+
+Request Examples
+================
+
+::
+
+    POST /v1/resources/count/
+    Host: 127.0.0.1:8999
+    User-Agent: keystoneauth1/2.3.0 python-requests/2.9.1 CPython/2.7.6
+    Accept: application/json
+    X-Auth-Token: 2b8882ba2ec44295bf300aecb2caa4f7
+
+
+Response Status code
+====================
+
+-  200 - OK
+-  404 - Bad request
+
+Response Body
+=============
+
+Returns counts of the requested resource, grouped by the selected field
+
+Query example
+=============
+
+::
+
+    POST /v1/resources/count/
+    Host: 135.248.19.18:8999
+    Content-Type: application/json
+    X-Auth-Token: 2b8882ba2ec44295bf300aecb2caa4f7
+
+    {
+        "query" :"
+         {
+            \"or\": [
+              {
+                \"==\": {
+                  \"state\": \"OK\"
+                }
+              },
+              {
+                \"==\": {
+                  \"state\": \"SUBOPTIMAL\"
+                }
+              }
+            ]
+
+         }",
+         "group_by" : "vitrage_operational_status",
+         "resource_type" : "nova.instance"
+         "all_tenants" : True
+     }
+
+
+Response Examples
+=================
+
+For the above request, will count all instances with status OK or SUBOPTIMAL,
+group by the status field.
+
+::
+
+    {
+      "OK": 157,
+      "SUBOPTIMAL": 3,
+    }
+
 Webhook List
 ^^^^^^^^^^^^
 List all webhooks.

releasenotes/notes/resource_count-1e3184e5f1f413ab.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - Resource count new API with support for queries and group-by.
+    Allows retrieving quick summaries of graph nodes.

releasenotes/notes/resource_list_query-b3fbcbe01090f64a.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - Resource list API now supports using a query
+deprecations:
+  - Resource list GET is deprecated, use POST instead.

vitrage/api/controllers/v1/alarm.py

@@ -35,7 +35,7 @@ LOG = log.getLogger(__name__)
 @profiler.trace_cls("alarm controller",
                     info={}, hide_args=False, trace_private=False)
 class AlarmsController(BaseAlarmsController):
-    count = count.CountsController()
+    count = count.AlarmCountsController()
     history = history.HistoryController()
 
     @pecan.expose('json')

vitrage/api/controllers/v1/count.py

@@ -25,7 +25,7 @@ LOG = log.getLogger(__name__)
 
 
 # noinspection PyBroadException
-class CountsController(RootRestController):
+class AlarmCountsController(RootRestController):
 
     @pecan.expose('json')
     def index(self, all_tenants=False):
@@ -53,3 +53,39 @@ class CountsController(RootRestController):
         except Exception:
             LOG.exception('failed to get alarm count.')
             abort(404, 'Failed to get alarm count.')
+
+
+class ResourceCountsController(RootRestController):
+
+    @pecan.expose('json')
+    def post(self, **kwargs):
+        resource_type = kwargs.get('resource_type', None)
+        all_tenants = kwargs.get('all_tenants', False)
+        all_tenants = bool_from_string(all_tenants)
+        query = kwargs.get('query')
+        group_by = kwargs.get('group_by')
+        if query:
+            query = json.loads(query)
+
+        if all_tenants:
+            enforce("count resources:all_tenants", pecan.request.headers,
+                    pecan.request.enforcer, {})
+        else:
+            enforce("count resources", pecan.request.headers,
+                    pecan.request.enforcer, {})
+
+        LOG.info('received get resource counts')
+
+        try:
+            resource_counts_json = pecan.request.client.call(
+                pecan.request.context, 'count_resources',
+                resource_type=resource_type,
+                all_tenants=all_tenants,
+                query=query,
+                group_by=group_by)
+
+            return json.loads(resource_counts_json)
+
+        except Exception:
+            LOG.exception('failed to get resource count.')
+            abort(404, 'Failed to get resource count.')

vitrage/api/controllers/v1/resource.py

@@ -13,11 +13,13 @@ import json
 import pecan
 
 from oslo_log import log
+from oslo_log import versionutils
 from oslo_utils.strutils import bool_from_string
 from osprofiler import profiler
 from pecan.core import abort
 
 from vitrage.api.controllers.rest import RootRestController
+from vitrage.api.controllers.v1 import count
 from vitrage.api.policy import enforce
 from vitrage.common.utils import decompress_obj
 
@@ -28,13 +30,47 @@ LOG = log.getLogger(__name__)
 @profiler.trace_cls("resource controller",
                     info={}, hide_args=False, trace_private=False)
 class ResourcesController(RootRestController):
+    count = count.ResourceCountsController()
+
     @pecan.expose('json')
+    def post(self, **kwargs):
+        LOG.info('post list resource with args: %s', kwargs)
+
+        resource_type = kwargs.get('resource_type', None)
+        all_tenants = kwargs.get('all_tenants', False)
+        all_tenants = bool_from_string(all_tenants)
+        query = kwargs.get('query')
+        try:
+            return self._get_resources(resource_type, all_tenants, query)
+        except Exception:
+            LOG.exception('Failed to list resources.')
+            abort(404, 'Failed to list resources.')
+
+    @pecan.expose('json')
+    @versionutils.deprecated(
+        as_of=versionutils.deprecated.STEIN,
+        what='rca:list_resources GET',
+        in_favor_of='rca:list_resources POST',
+    )
     def get_all(self, **kwargs):
         LOG.info('get list resource with args: %s', kwargs)
 
         resource_type = kwargs.get('resource_type', None)
         all_tenants = kwargs.get('all_tenants', False)
         all_tenants = bool_from_string(all_tenants)
+        query = kwargs.get('query')
+
+        try:
+            return self._get_resources(resource_type, all_tenants, query)
+        except Exception:
+            LOG.exception('Failed to list resources.')
+            abort(404, 'Failed to list resources.')
+
+    @staticmethod
+    def _get_resources(resource_type=None, all_tenants=False, query=None):
+        if query:
+            query = json.loads(query)
+
         if all_tenants:
             enforce('list resources:all_tenants', pecan.request.headers,
                     pecan.request.enforcer, {})
@@ -42,29 +78,17 @@ class ResourcesController(RootRestController):
             enforce('list resources', pecan.request.headers,
                     pecan.request.enforcer, {})
 
-        LOG.info('received resources list with filter %s', resource_type)
+        LOG.info('get resources with type: %s, all_tenants: %s, query: %s',
+                 resource_type, all_tenants, str(query))
 
-        try:
+        resources = pecan.request.client.call(
-            return self._get_resources(resource_type, all_tenants)
+            pecan.request.context,
-        except Exception:
+            'get_resources',
-            LOG.exception('Failed to list resources.')
+            resource_type=resource_type,
-            abort(404, 'Failed to list resources.')
+            all_tenants=all_tenants,
-
+            query=query)
-    @staticmethod
+        resources = decompress_obj(resources)['resources']
-    def _get_resources(resource_type=None, all_tenants=False):
+        return resources
-        LOG.info('get_resources with type: %s, all_tenants: %s',
-                 resource_type, all_tenants)
-        try:
-            resources = \
-                pecan.request.client.call(pecan.request.context,
-                                          'get_resources',
-                                          resource_type=resource_type,
-                                          all_tenants=all_tenants)
-            resources = decompress_obj(resources)['resources']
-            return resources
-        except Exception:
-            LOG.exception('Failed to get resources.')
-            abort(404, 'Failed to list resources.')
 
     @pecan.expose('json')
     def get(self, vitrage_id):

vitrage/api_handler/apis/resource.py

@@ -36,10 +36,41 @@ class ResourceApis(base.EntityGraphApisBase):
 
     @timed_method(log_results=True)
     @base.lock_graph
-    def get_resources(self, ctx, resource_type=None, all_tenants=False):
+    def get_resources(self, ctx, resource_type=None, all_tenants=False,
-        LOG.debug('ResourceApis get_resources - resource_type: %s,'
+                      query=None):
-                  'all_tenants: %s', str(resource_type), all_tenants)
+        LOG.debug(
+            'ResourceApis get_resources - resource_type: %s, all_tenants: %s,'
+            ' query: %s',
+            str(resource_type),
+            all_tenants,
+            str(query))
 
+        query = self._get_query(ctx, resource_type, all_tenants, query)
+        resources = self.entity_graph.get_vertices(query_dict=query)
+        data = {'resources': [r.properties for r in resources]}
+        return compress_obj(data, level=1)
+
+    @timed_method(log_results=True)
+    @base.lock_graph
+    def count_resources(self, ctx, resource_type=None, all_tenants=False,
+                        query=None, group_by=None):
+        LOG.debug(
+            'ResourceApis count_resources - type: %s, all_tenants: %s,'
+            ' query: %s, group_by: %s',
+            str(resource_type),
+            all_tenants,
+            str(query),
+            str(group_by))
+
+        query = self._get_query(ctx, resource_type, all_tenants, query)
+        if group_by is None:
+            group_by = VProps.VITRAGE_TYPE
+        counts = self.entity_graph.get_vertices_count(query_dict=query,
+                                                      group_by=group_by)
+
+        return json.dumps(counts)
+
+    def _get_query(self, ctx, resource_type, all_tenants, query_dict):
         project_id = ctx.get(TenantProps.TENANT, None)
         is_admin_project = ctx.get(TenantProps.IS_ADMIN, False)
 
@@ -56,9 +87,9 @@ class ResourceApis(base.EntityGraphApisBase):
             type_query = {'==': {VProps.VITRAGE_TYPE: resource_type}}
             query['and'].append(type_query)
 
-        resources = self.entity_graph.get_vertices(query_dict=query)
+        if query_dict:
-        data = {'resources': [r.properties for r in resources]}
+            query['and'].append(query_dict)
-        return compress_obj(data, level=1)
+        return query
 
     @base.lock_graph
     def show_resource(self, ctx, vitrage_id):

vitrage/common/policies/resource.py

@@ -50,6 +50,31 @@ rules = [
                 'method': 'GET'
             }
         ]
+    ),
+    policy.DocumentedRuleDefault(
+        name='count resources',
+        check_str=base.UNPROTECTED,
+        description='count the resources with the specified type, or all the '
+                    'resources',
+        operations=[
+            {
+                'path': '/resources/count',
+                'method': 'GET'
+            }
+        ]
+    ),
+    policy.DocumentedRuleDefault(
+        name='count resources:all_tenants',
+        check_str=base.ROLE_ADMIN,
+        description='Count the resources with the specified type, or all the '
+                    'resources. Include resources of all tenants (if the user'
+                    ' has the permissions)',
+        operations=[
+            {
+                'path': '/resources/count',
+                'method': 'GET'
+            }
+        ]
     )
 ]
 

vitrage/graph/driver/networkx_graph.py

@@ -12,6 +12,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+from collections import defaultdict
 import copy
 import json
 import networkx as nx
@@ -262,6 +263,14 @@ class NXGraph(Graph):
                 vertices_ids.add(node)
         return vertices_ids
 
+    def get_vertices_count(self, query_dict, group_by):
+        vertices_counts = defaultdict(int)
+        match_func = create_predicate(query_dict) if query_dict else None
+        for node, node_data in self._g.nodes(data=True):
+            if match_func is None or match_func(node_data):
+                vertices_counts[node_data.get(group_by, '')] += 1
+        return vertices_counts
+
     def get_vertices_by_key(self, key_values_hash):
 
         if key_values_hash in self.key_to_vertex_ids:

vitrage/tests/functional/api_handler/test_apis.py

@@ -31,6 +31,7 @@ from vitrage.common.utils import decompress_obj
 from vitrage.datasources import NOVA_HOST_DATASOURCE
 from vitrage.datasources import NOVA_INSTANCE_DATASOURCE
 from vitrage.datasources import NOVA_ZONE_DATASOURCE
+from vitrage.datasources import OPENSTACK_CLUSTER
 from vitrage.datasources.transformer_base \
     import create_cluster_placeholder_vertex
 from vitrage.entity_graph.mappings.operational_alarm_severity import \
@@ -268,6 +269,23 @@ class TestApis(TestEntityGraphUnitBase, TestConfiguration):
         # Test assertions
         self.assertThat(resources, matchers.HasLength(5))
 
+    def test_resource_list_with_admin_project_and_query(self):
+        # Setup
+        graph = self._create_graph()
+        apis = ResourceApis(graph, None, self.api_lock)
+        ctx = {'tenant': 'project_2', 'is_admin': True}
+
+        # Action
+        resources = apis.get_resources(
+            ctx,
+            resource_type=NOVA_INSTANCE_DATASOURCE,
+            all_tenants=False,
+            query={'==': {'id': 'instance_3'}})
+        resources = decompress_obj(resources)['resources']
+
+        # Test assertions
+        self.assertThat(resources, matchers.HasLength(1))
+
     def test_resource_list_with_not_admin_project(self):
         # Setup
         graph = self._create_graph()
@@ -332,6 +350,128 @@ class TestApis(TestEntityGraphUnitBase, TestConfiguration):
         # Test assertions
         self.assertThat(resources, matchers.HasLength(7))
 
+    def test_resource_count_with_admin_project(self):
+        # Setup
+        graph = self._create_graph()
+        apis = ResourceApis(graph, None, self.api_lock)
+        ctx = {'tenant': 'project_2', 'is_admin': True}
+
+        # Action
+        resources = apis.count_resources(
+            ctx,
+            resource_type=None,
+            all_tenants=False)
+        resources = json.loads(resources)
+
+        # Test assertions
+        self.assertEqual(2, resources[NOVA_INSTANCE_DATASOURCE])
+        self.assertEqual(1, resources[NOVA_ZONE_DATASOURCE])
+        self.assertEqual(1, resources[OPENSTACK_CLUSTER])
+        self.assertEqual(1, resources[NOVA_HOST_DATASOURCE])
+
+    def test_resource_count_with_admin_project_and_query(self):
+        # Setup
+        graph = self._create_graph()
+        apis = ResourceApis(graph, None, self.api_lock)
+        ctx = {'tenant': 'project_2', 'is_admin': True}
+
+        # Action
+        resources = apis.count_resources(
+            ctx,
+            resource_type=NOVA_INSTANCE_DATASOURCE,
+            all_tenants=False,
+            query={'==': {'id': 'instance_3'}})
+        resources = json.loads(resources)
+
+        # Test assertions
+        self.assertEqual(1, resources[NOVA_INSTANCE_DATASOURCE])
+
+    def test_resource_count_with_not_admin_project(self):
+        # Setup
+        graph = self._create_graph()
+        apis = ResourceApis(graph, None, self.api_lock)
+        ctx = {'tenant': 'project_2', 'is_admin': False}
+
+        # Action
+        resources = apis.count_resources(
+            ctx,
+            resource_type=None,
+            all_tenants=False)
+        resources = json.loads(resources)
+
+        # Test assertions
+        self.assertEqual(2, resources[NOVA_INSTANCE_DATASOURCE])
+
+    def test_resource_count_with_not_admin_project_and_no_existing_type(self):
+        # Setup
+        graph = self._create_graph()
+        apis = ResourceApis(graph, None, self.api_lock)
+        ctx = {'tenant': 'project_2', 'is_admin': False}
+
+        # Action
+        resources = apis.count_resources(
+            ctx,
+            resource_type=NOVA_HOST_DATASOURCE,
+            all_tenants=False)
+        resources = json.loads(resources)
+
+        # Test assertions
+        self.assertThat(resources.items(), matchers.HasLength(0))
+
+    def test_resource_count_with_not_admin_project_and_existing_type(self):
+        # Setup
+        graph = self._create_graph()
+        apis = ResourceApis(graph, None, self.api_lock)
+        ctx = {'tenant': 'project_2', 'is_admin': False}
+
+        # Action
+        resources = apis.count_resources(
+            ctx,
+            resource_type=NOVA_INSTANCE_DATASOURCE,
+            all_tenants=False)
+        resources = json.loads(resources)
+
+        # Test assertions
+        self.assertEqual(2, resources[NOVA_INSTANCE_DATASOURCE])
+
+    def test_resource_count_with_all_tenants(self):
+        # Setup
+        graph = self._create_graph()
+        apis = ResourceApis(graph, None, self.api_lock)
+        ctx = {'tenant': 'project_1', 'is_admin': False}
+
+        # Action
+        resources = apis.count_resources(
+            ctx,
+            resource_type=None,
+            all_tenants=True)
+        resources = json.loads(resources)
+
+        # Test assertions
+        self.assertEqual(4, resources[NOVA_INSTANCE_DATASOURCE])
+        self.assertEqual(1, resources[NOVA_ZONE_DATASOURCE])
+        self.assertEqual(1, resources[OPENSTACK_CLUSTER])
+        self.assertEqual(1, resources[NOVA_HOST_DATASOURCE])
+
+    def test_resource_count_with_all_tenants_and_group_by(self):
+        # Setup
+        graph = self._create_graph()
+        apis = ResourceApis(graph, None, self.api_lock)
+        ctx = {'tenant': 'project_1', 'is_admin': False}
+
+        # Action
+        resources = apis.count_resources(
+            ctx,
+            resource_type=None,
+            all_tenants=True,
+            group_by=VProps.PROJECT_ID)
+        resources = json.loads(resources)
+
+        # Test assertions
+        self.assertEqual(2, resources['project_1'])
+        self.assertEqual(2, resources['project_2'])
+        self.assertEqual(3, resources[''])
+
     def test_resource_show_with_admin_and_no_project_resource(self):
         # Setup
         graph = self._create_graph()