Implements "not" operator for complex query

Change-Id: Idf17b35c5f4267b9254a64e37b0d8b1b0dcbca89
Implements: blueprint complex-filter-expressions-in-api-queries

ceilometer/api/controllers/v2.py

@@ -1115,15 +1115,24 @@ class ValidatedComplexQuery(object):
             "minProperties": 1,
             "maxProperties": 1}
 
+        schema_not = {
+            "type": "object",
+            "patternProperties": {"(?i)^not$": {"$ref": "#"}},
+            "additionalProperties": False,
+            "minProperties": 1,
+            "maxProperties": 1}
+
         self.schema = {
             "oneOf": [{"$ref": "#/definitions/leaf_simple_ops"},
                       {"$ref": "#/definitions/leaf_in"},
-                      {"$ref": "#/definitions/and_or"}],
+                      {"$ref": "#/definitions/and_or"},
+                      {"$ref": "#/definitions/not"}],
             "minProperties": 1,
             "maxProperties": 1,
             "definitions": {"leaf_simple_ops": schema_leaf_simple_ops,
                             "leaf_in": schema_leaf_in,
-                            "and_or": schema_and_or}}
+                            "and_or": schema_and_or,
+                            "not": schema_not}}
 
         self.orderby_schema = {
             "type": "array",
@@ -1184,6 +1193,8 @@ class ValidatedComplexQuery(object):
         if op.lower() in self.complex_operators:
             for i, operand in enumerate(tree[op]):
                 self._traverse_postorder(operand, visitor)
+        if op.lower() == "not":
+            self._traverse_postorder(tree[op], visitor)
 
         visitor(tree)
 

ceilometer/storage/impl_sqlalchemy.py

@@ -27,6 +27,7 @@ from sqlalchemy import and_
 from sqlalchemy import asc
 from sqlalchemy import desc
 from sqlalchemy import func
+from sqlalchemy import not_
 from sqlalchemy import or_
 from sqlalchemy.orm import aliased
 
@@ -1207,7 +1208,8 @@ class QueryTransformer(object):
                  "in": lambda field_name, values: field_name.in_(values)}
 
     complex_operators = {"or": or_,
-                         "and": and_}
+                         "and": and_,
+                         "not": not_}
 
     ordering_functions = {"asc": asc,
                           "desc": desc}
@@ -1218,6 +1220,8 @@ class QueryTransformer(object):
 
     def _handle_complex_op(self, complex_op, nodes):
         op = self.complex_operators[complex_op]
+        if op == not_:
+            nodes = [nodes]
         element_list = []
         for node in nodes:
             element = self._transform(node)

ceilometer/storage/pymongo_base.py

@@ -130,21 +130,6 @@ class Connection(base.Connection):
     """Base Connection class for MongoDB and DB2 drivers.
     """
 
-    operators = {"<": "$lt",
-                 ">": "$gt",
-                 "<=": "$lte",
-                 "=<": "$lte",
-                 ">=": "$gte",
-                 "=>": "$gte",
-                 "!=": "$ne",
-                 "in": "$in"}
-
-    complex_operators = {"or": "$or",
-                         "and": "$and"}
-
-    ordering_functions = {"asc": pymongo.ASCENDING,
-                          "desc": pymongo.DESCENDING}
-
     def get_users(self, source=None):
         """Return an iterable of user id strings.
 
@@ -288,10 +273,11 @@ class Connection(base.Connection):
             return []
         query_filter = {}
         orderby_filter = [("timestamp", pymongo.DESCENDING)]
+        transformer = QueryTransformer()
         if orderby is not None:
-            orderby_filter = self._transform_orderby(orderby)
+            orderby_filter = transformer.transform_orderby(orderby)
         if filter_expr is not None:
-            query_filter = self._transform_filter(filter_expr)
+            query_filter = transformer.transform_filter(filter_expr)
 
         retrieve = {models.Meter: self._retrieve_samples,
                     models.Alarm: self._retrieve_alarms,
@@ -348,45 +334,6 @@ class Connection(base.Connection):
             del ah['_id']
             yield models.AlarmChange(**ah)
 
-    def _transform_orderby(self, orderby):
-        orderby_filter = []
-
-        for field in orderby:
-            field_name = field.keys()[0]
-            ordering = self.ordering_functions[field.values()[0]]
-            orderby_filter.append((field_name, ordering))
-        return orderby_filter
-
-    def _transform_filter(self, condition):
-
-        def process_json_tree(condition_tree):
-            operator_node = condition_tree.keys()[0]
-            nodes = condition_tree.values()[0]
-
-            if operator_node in self.complex_operators:
-                element_list = []
-                for node in nodes:
-                    element = process_json_tree(node)
-                    element_list.append(element)
-                complex_operator = self.complex_operators[operator_node]
-                op = {complex_operator: element_list}
-                return op
-            else:
-                field_name = nodes.keys()[0]
-                field_value = nodes.values()[0]
-                # no operator for equal in Mongo
-                if operator_node == "=":
-                    op = {field_name: field_value}
-                    return op
-                if operator_node in self.operators:
-                    operator = self.operators[operator_node]
-                    op = {
-                        field_name: {
-                            operator: field_value}}
-                    return op
-
-        return process_json_tree(condition)
-
     @classmethod
     def _ensure_encapsulated_rule_format(cls, alarm):
         """This ensure the alarm returned by the storage have the correct
@@ -449,3 +396,124 @@ class Connection(base.Connection):
             for elem in matching_metadata:
                 new_matching_metadata[elem['key']] = elem['value']
             return new_matching_metadata
+
+
+class QueryTransformer(object):
+
+    operators = {"<": "$lt",
+                 ">": "$gt",
+                 "<=": "$lte",
+                 "=<": "$lte",
+                 ">=": "$gte",
+                 "=>": "$gte",
+                 "!=": "$ne",
+                 "in": "$in"}
+
+    complex_operators = {"or": "$or",
+                         "and": "$and"}
+
+    ordering_functions = {"asc": pymongo.ASCENDING,
+                          "desc": pymongo.DESCENDING}
+
+    def transform_orderby(self, orderby):
+        orderby_filter = []
+
+        for field in orderby:
+            field_name = field.keys()[0]
+            ordering = self.ordering_functions[field.values()[0]]
+            orderby_filter.append((field_name, ordering))
+        return orderby_filter
+
+    @staticmethod
+    def _move_negation_to_leaf(condition):
+        """Moves every not operator to the leafs by
+        applying the De Morgan rules and anihilating
+        double negations
+        """
+        def _apply_de_morgan(tree, negated_subtree, negated_op):
+            if negated_op == "and":
+                new_op = "or"
+            else:
+                new_op = "and"
+
+            tree[new_op] = [{"not": child}
+                            for child in negated_subtree[negated_op]]
+            del tree["not"]
+
+        def transform(subtree):
+            op = subtree.keys()[0]
+            if op in ["and", "or"]:
+                [transform(child) for child in subtree[op]]
+            elif op == "not":
+                negated_tree = subtree[op]
+                negated_op = negated_tree.keys()[0]
+                if negated_op == "and":
+                    _apply_de_morgan(subtree, negated_tree, negated_op)
+                    transform(subtree)
+                elif negated_op == "or":
+                    _apply_de_morgan(subtree, negated_tree, negated_op)
+                    transform(subtree)
+                elif negated_op == "not":
+                    # two consecutive not annihilates theirselves
+                    new_op = negated_tree.values()[0].keys()[0]
+                    subtree[new_op] = negated_tree[negated_op][new_op]
+                    del subtree["not"]
+                    transform(subtree)
+
+        transform(condition)
+
+    def transform_filter(self, condition):
+        # in Mongo not operator can only be applied to
+        # simple expressions so we have to move every
+        # not operator to the leafs of the expression tree
+        self._move_negation_to_leaf(condition)
+        return self._process_json_tree(condition)
+
+    def _handle_complex_op(self, complex_op, nodes):
+        element_list = []
+        for node in nodes:
+            element = self._process_json_tree(node)
+            element_list.append(element)
+        complex_operator = self.complex_operators[complex_op]
+        op = {complex_operator: element_list}
+        return op
+
+    def _handle_not_op(self, negated_tree):
+        # assumes that not is moved to the leaf already
+        # so we are next to a leaf
+        negated_op = negated_tree.keys()[0]
+        negated_field = negated_tree[negated_op].keys()[0]
+        value = negated_tree[negated_op][negated_field]
+        if negated_op == "=":
+            return {negated_field: {"$ne": value}}
+        elif negated_op == "!=":
+            return {negated_field: value}
+        else:
+            return {negated_field: {"$not":
+                                    {self.operators[negated_op]: value}}}
+
+    def _handle_simple_op(self, simple_op, nodes):
+        field_name = nodes.keys()[0]
+        field_value = nodes.values()[0]
+
+        # no operator for equal in Mongo
+        if simple_op == "=":
+            op = {field_name: field_value}
+            return op
+
+        operator = self.operators[simple_op]
+        op = {field_name: {operator: field_value}}
+        return op
+
+    def _process_json_tree(self, condition_tree):
+        operator_node = condition_tree.keys()[0]
+        nodes = condition_tree.values()[0]
+
+        if operator_node in self.complex_operators:
+            return self._handle_complex_op(operator_node, nodes)
+
+        if operator_node == "not":
+            negated_tree = condition_tree[operator_node]
+            return self._handle_not_op(negated_tree)
+
+        return self._handle_simple_op(operator_node, nodes)

ceilometer/tests/api/v2/test_complex_query.py

@@ -360,3 +360,30 @@ class TestFilterSyntaxValidation(test.BaseTestCase):
         self.assertRaises(jsonschema.ValidationError,
                           self.query._validate_filter,
                           filter)
+
+    def test_not(self):
+        filter = {"not": {"=": {"project_id": "value"}}}
+        self.query._validate_filter(filter)
+
+        filter = {
+            "not":
+            {"or":
+             [{"and":
+               [{"=": {"project_id": "string_value"}},
+                {"=": {"resource_id": "value"}},
+                {"<": {"counter_name": 42}}]},
+              {"=": {"counter_name": "value"}}]}}
+        self.query._validate_filter(filter)
+
+    def test_not_with_zero_child_is_invalid(self):
+        filter = {"not": {}}
+        self.assertRaises(jsonschema.ValidationError,
+                          self.query._validate_filter,
+                          filter)
+
+    def test_not_with_more_than_one_child_is_invalid(self):
+        filter = {"not": {"=": {"project_id": "value"},
+                          "!=": {"resource_id": "value"}}}
+        self.assertRaises(jsonschema.ValidationError,
+                          self.query._validate_filter,
+                          filter)

ceilometer/tests/api/v2/test_complex_query_scenarios.py

@@ -256,6 +256,15 @@ class TestQueryMetersController(tests_api.FunctionalTest,
         for sample in data.json:
             self.assertTrue(sample["metadata"]["util"] >= 0.5)
 
+    def test_filter_with_negation(self):
+        filter_expr = '{"not": {">=": {"metadata.util": 0.5}}}'
+        data = self.post_json(self.url,
+                              params={"filter": filter_expr})
+
+        self.assertEqual(1, len(data.json))
+        for sample in data.json:
+            self.assertTrue(float(sample["metadata"]["util"]) < 0.5)
+
     def test_limit_should_be_positive(self):
         data = self.post_json(self.url,
                               params={"limit": 0},

ceilometer/tests/storage/test_storage_scenarios.py

@@ -728,20 +728,21 @@ class ComplexSampleQueryTest(DBTestBase,
                              tests_db.MixinTestsWithBackendScenarios):
     def setUp(self):
         super(ComplexSampleQueryTest, self).setUp()
-        self.complex_filter = {"and":
+        self.complex_filter = {
-                               [{"or":
+            "and":
-                                 [{"=": {"resource_id": "resource-id-42"}},
+            [{"or":
-                                  {"=": {"resource_id": "resource-id-44"}}]},
+              [{"=": {"resource_id": "resource-id-42"}},
-                                {"and":
+               {"=": {"resource_id": "resource-id-44"}}]},
-                                 [{"=": {"counter_name": "cpu_util"}},
+             {"and":
-                                  {"and":
+              [{"=": {"counter_name": "cpu_util"}},
-                                   [{">": {"counter_volume": 0.4}},
+               {"and":
-                                    {"<=": {"counter_volume": 0.8}}]}]}]}
+                [{">": {"counter_volume": 0.4}},
+                 {"not": {">": {"counter_volume": 0.8}}}]}]}]}
         or_expression = [{"=": {"resource_id": "resource-id-42"}},
                          {"=": {"resource_id": "resource-id-43"}},
                          {"=": {"resource_id": "resource-id-44"}}]
         and_expression = [{">": {"counter_volume": 0.4}},
-                          {"<=": {"counter_volume": 0.8}}]
+                          {"not": {">": {"counter_volume": 0.8}}}]
         self.complex_filter_list = {"and":
                                     [{"or": or_expression},
                                      {"and":
@@ -1017,6 +1018,95 @@ class ComplexSampleQueryTest(DBTestBase,
         results = list(self.conn.query_samples(filter_expr=filter_expr))
         self.assertEqual(len(results), 0)
 
+    def test_query_negated_metadata(self):
+        self._create_samples()
+
+        filter_expr = {
+            "and": [{"=": {"resource_id": "resource-id-42"}},
+                    {"not": {"or": [{">": {"resource_metadata.an_int_key":
+                                           43}},
+                                    {"<=": {"resource_metadata.a_float_key":
+                                            0.41}}]}}]}
+
+        results = list(self.conn.query_samples(filter_expr=filter_expr))
+
+        self.assertEqual(len(results), 3)
+        for sample in results:
+            self.assertEqual(sample.resource_id, "resource-id-42")
+            self.assertTrue(sample.resource_metadata["an_int_key"] <= 43)
+            self.assertTrue(sample.resource_metadata["a_float_key"] > 0.41)
+
+    def test_query_negated_complex_expression(self):
+        self._create_samples()
+        filter_expr = {
+            "and":
+            [{"=": {"counter_name": "cpu_util"}},
+             {"not":
+              {"or":
+               [{"or":
+                 [{"=": {"resource_id": "resource-id-42"}},
+                  {"=": {"resource_id": "resource-id-44"}}]},
+                {"and":
+                 [{">": {"counter_volume": 0.4}},
+                  {"<": {"counter_volume": 0.8}}]}]}}]}
+
+        results = list(self.conn.query_samples(filter_expr=filter_expr))
+
+        self.assertEqual(len(results), 4)
+        for sample in results:
+            self.assertEqual(sample.resource_id,
+                             "resource-id-43")
+            self.assertIn(sample.counter_volume, [0.39, 0.4, 0.8, 0.81])
+            self.assertEqual(sample.counter_name,
+                             "cpu_util")
+
+    def test_query_with_double_negation(self):
+        self._create_samples()
+        filter_expr = {
+            "and":
+            [{"=": {"counter_name": "cpu_util"}},
+             {"not":
+              {"or":
+               [{"or":
+                 [{"=": {"resource_id": "resource-id-42"}},
+                  {"=": {"resource_id": "resource-id-44"}}]},
+                {"and": [{"not": {"<=": {"counter_volume": 0.4}}},
+                         {"<": {"counter_volume": 0.8}}]}]}}]}
+
+        results = list(self.conn.query_samples(filter_expr=filter_expr))
+
+        self.assertEqual(len(results), 4)
+        for sample in results:
+            self.assertEqual(sample.resource_id,
+                             "resource-id-43")
+            self.assertIn(sample.counter_volume, [0.39, 0.4, 0.8, 0.81])
+            self.assertEqual(sample.counter_name,
+                             "cpu_util")
+
+    def test_query_negate_not_equal(self):
+        self._create_samples()
+        filter_expr = {"not": {"!=": {"resource_id": "resource-id-43"}}}
+
+        results = list(self.conn.query_samples(filter_expr=filter_expr))
+
+        self.assertEqual(len(results), 6)
+        for sample in results:
+            self.assertEqual(sample.resource_id,
+                             "resource-id-43")
+
+    def test_query_negated_in_op(self):
+        self._create_samples()
+        filter_expr = {
+            "and": [{"not": {"in": {"counter_volume": [0.39, 0.4, 0.79]}}},
+                    {"=": {"resource_id": "resource-id-42"}}]}
+
+        results = list(self.conn.query_samples(filter_expr=filter_expr))
+
+        self.assertEqual(len(results), 3)
+        for sample in results:
+            self.assertIn(sample.counter_volume,
+                          [0.41, 0.8, 0.81])
+
 
 class StatisticsTest(DBTestBase,
                      tests_db.MixinTestsWithBackendScenarios):

doc/source/webapi/v2.rst

@@ -96,8 +96,18 @@ Complex Query
 The filter expressions of the Complex Query feature operate on the fields
 of *Sample*, *Alarm* and *AlarmChange*. The following comparison operators are
 supported: *=*, *!=*, *<*, *<=*, *>*, *>=* and *in*; and the following logical
-operators can be used: *and* and *or*. The field names are validated against
+operators can be used: *and* *or* and *not*. The field names are validated
-the database models.
+against the database models.
+
+.. note:: The *not* operator has different meaning in Mongo DB and in SQL DB engine.
+   If the *not* operator is applied on a non existent metadata field then
+   the result depends on the DB engine. For example if
+   {"not": {"metadata.nonexistent_field" : "some value"}} filter is used in a query
+   the Mongo DB will return every Sample object as *not* operator evaluated true
+   for every Sample where the given field does not exists. See more in the Mongod DB doc.
+   In the other hand SQL based DB engine will return empty result as the join operation
+   on the metadata table will return zero row as the on clause of the join which
+   tries to match on the metadata field name is never fulfilled.
 
 Complex Query supports defining the list of orderby expressions in the form
 of [{"field_name": "asc"}, {"field_name2": "desc"}, ...].
@@ -418,13 +428,14 @@ to the /v2/query/samples endpoint of Ceilometer API using POST request.
 
 To check for *cpu_util* samples reported between 18:00-18:15 or between 18:30 - 18:45
 on a particular date (2013-12-01), where the utilization is between 23 and 26 percent,
-the following filter expression can be created::
+but not exactly 25.12 percent, the following filter expression can be created::
 
     {"and":
      [{"and":
       [{"=": {"counter_name": "cpu_util"}},
        {">": {"counter_volume": 0.23}},
-       {"<": {"counter_volume": 0.26}}]},
+       {"<": {"counter_volume": 0.26}},
+       {"not": {"=": {"counter_volume": 0.2512}}}]},
       {"or":
        [{"and":
         [{">": {"timestamp": "2013-12-01T18:00:00"}},
@@ -446,7 +457,7 @@ By adding a limit criteria to the request, which maximizes the number of returne
 to four, the query looks like the following::
 
     {
-    "filter" : "{\"and\":[{\"and\": [{\"=\": {\"counter_name\": \"cpu_util\"}}, {\">\": {\"counter_volume\": 0.23}}, {\"<\": {\"counter_volume\": 0.26}}]}, {\"or\": [{\"and\": [{\">\": {\"timestamp\": \"2013-12-01T18:00:00\"}}, {\"<\": {\"timestamp\": \"2013-12-01T18:15:00\"}}]}, {\"and\": [{\">\": {\"timestamp\": \"2013-12-01T18:30:00\"}}, {\"<\": {\"timestamp\": \"2013-12-01T18:45:00\"}}]}]}]}",
+    "filter" : "{\"and\":[{\"and\": [{\"=\": {\"counter_name\": \"cpu_util\"}}, {\">\": {\"counter_volume\": 0.23}}, {\"<\": {\"counter_volume\": 0.26}}, {\"not\": {\"=\": {\"counter_volume\": 0.2512}}}]}, {\"or\": [{\"and\": [{\">\": {\"timestamp\": \"2013-12-01T18:00:00\"}}, {\"<\": {\"timestamp\": \"2013-12-01T18:15:00\"}}]}, {\"and\": [{\">\": {\"timestamp\": \"2013-12-01T18:30:00\"}}, {\"<\": {\"timestamp\": \"2013-12-01T18:45:00\"}}]}]}]}",
     "orderby" : "[{\"counter_volume\": \"ASC\"}, {\"timestamp\": \"DESC\"}]",
     "limit" : 4
     }