Merge "Implements complex query functionality for alarms"

ceilometer/api/controllers/v2.py

@@ -3,11 +3,14 @@
 # Copyright © 2012 New Dream Network, LLC (DreamHost)
 # Copyright 2013 IBM Corp.
 # Copyright © 2013 eNovance <licensing@enovance.com>
+# Copyright Ericsson AB 2013. All rights reserved
 #
 # Authors: Doug Hellmann <doug.hellmann@dreamhost.com>
 #          Angus Salkeld <asalkeld@redhat.com>
 #          Eoghan Glynn <eglynn@redhat.com>
 #          Julien Danjou <julien@danjou.info>
+#          Ildiko Vancsa <ildiko.vancsa@ericsson.com>
+#          Balazs Gibizer <balazs.gibizer@ericsson.com>
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
@@ -1058,7 +1061,7 @@ class ValidatedComplexQuery(object):
             "minProperties": 1,
             "maxProperties": 1}}
 
-    timestamp_fields = ["timestamp"]
+    timestamp_fields = ["timestamp", "state_timestamp"]
 
     def __init__(self, query):
         self.original_query = query
@@ -2049,8 +2052,28 @@ class QuerySamplesController(rest.RestController):
                                             query.limit)]
 
 
+class QueryAlarmsController(rest.RestController):
+    """Provides complex query possibilities for alarms
+    """
+    @wsme_pecan.wsexpose([Alarm], body=ComplexQuery)
+    def post(self, body):
+        """Define query for retrieving Alarm data.
+
+        :param body: Query rules for the alarms to be returned.
+        """
+        query = ValidatedComplexQuery(body)
+        query.validate(visibility_field="project_id")
+        conn = pecan.request.storage_conn
+        return [Alarm.from_db_model(s)
+                for s in conn.query_alarms(query.filter_expr,
+                                           query.orderby,
+                                           query.limit)]
+
+
 class QueryController(rest.RestController):
+
     samples = QuerySamplesController()
+    alarms = QueryAlarmsController()
 
 
 class V2Controller(object):

ceilometer/storage/base.py

@@ -331,3 +331,15 @@ class Connection(object):
 
         raise NotImplementedError(_('Complex query for samples \
             is not implemented.'))
+
+    @staticmethod
+    def query_alarms(filter_expr=None, orderby=None, limit=None):
+        """Return an iterable of model.Alarm objects.
+
+        :param filter_expr: Filter expression for query.
+        :param orderby: List of field name and direction pairs for order by.
+        :param limit: Maximum number of results to return.
+        """
+
+        raise NotImplementedError(_('Complex query for alarms \
+            is not implemented.'))

ceilometer/storage/impl_mongodb.py

@@ -779,7 +779,7 @@ class Connection(base.Connection):
                                       [("timestamp", pymongo.DESCENDING)],
                                       limit)
 
-    def query_samples(self, filter_expr=None, orderby=None, limit=None):
+    def _retrieve_data(self, filter_expr, orderby, limit, model):
         if limit == 0:
             return []
         query_filter = {}
@@ -790,7 +790,12 @@ class Connection(base.Connection):
             query_filter = self._transform_filter(
                 filter_expr)
 
-        return self._retrieve_samples(query_filter, orderby_filter, limit)
+        retrieve = {models.Meter: self._retrieve_samples,
+                    models.Alarm: self._retrieve_alarms}
+        return retrieve[model](query_filter, orderby_filter, limit)
+
+    def query_samples(self, filter_expr=None, orderby=None, limit=None):
+        return self._retrieve_data(filter_expr, orderby, limit, models.Meter)
 
     def _transform_orderby(self, orderby):
         orderby_filter = []
@@ -944,6 +949,22 @@ class Connection(base.Connection):
         del alarm['matching_metadata']
         alarm['rule']['query'] = query
 
+    def _retrieve_alarms(self, query_filter, orderby, limit):
+        if limit is not None:
+            alarms = self.db.alarm.find(query_filter,
+                                        limit=limit,
+                                        sort=orderby)
+        else:
+            alarms = self.db.alarm.find(
+                query_filter, sort=orderby)
+
+        for alarm in alarms:
+            a = {}
+            a.update(alarm)
+            del a['_id']
+            self._ensure_encapsulated_rule_format(a)
+            yield models.Alarm(**a)
+
     def get_alarms(self, name=None, user=None,
                    project=None, enabled=None, alarm_id=None, pagination=None):
         """Yields a lists of alarms that match filters
@@ -969,12 +990,7 @@ class Connection(base.Connection):
         if alarm_id is not None:
             q['alarm_id'] = alarm_id
 
-        for alarm in self.db.alarm.find(q):
-            a = {}
-            a.update(alarm)
-            del a['_id']
-            self._ensure_encapsulated_rule_format(a)
-            yield models.Alarm(**a)
+        return self._retrieve_alarms(q, [], None)
 
     def update_alarm(self, alarm):
         """update alarm
@@ -1052,3 +1068,8 @@ class Connection(base.Connection):
         """Record alarm change event.
         """
         self.db.alarm_history.insert(alarm_change)
+
+    def query_alarms(self, filter_expr=None, orderby=None, limit=None):
+        """Return an iterable of model.Alarm objects.
+        """
+        return self._retrieve_data(filter_expr, orderby, limit, models.Alarm)

ceilometer/storage/impl_sqlalchemy.py

@@ -532,10 +532,13 @@ class Connection(base.Connection):
                 source=resource.sources[0].id,
                 user_id=resource.user_id)
 
-    def _retrieve_samples(self, query, orderby, limit, table):
+    def _apply_options(self, query, orderby, limit, table):
         query = self._apply_order_by(query, orderby, table)
         if limit is not None:
             query = query.limit(limit)
+        return query
+
+    def _retrieve_samples(self, query):
         samples = query.all()
 
         for s in samples:
@@ -575,13 +578,16 @@ class Connection(base.Connection):
         query = make_query_from_filter(session, query, sample_filter,
                                        require_meter=False)
 
-        return self._retrieve_samples(query, None, limit, table)
+        query = self._apply_options(query,
+                                    None,
+                                    limit,
+                                    table)
+        return self._retrieve_samples(query)
 
-    def query_samples(self, filter_expr=None, orderby=None, limit=None):
+    def _retrieve_data(self, filter_expr, orderby, limit, table):
         if limit == 0:
             return []
 
-        table = models.Meter
         session = self._get_db_session()
         query = session.query(table)
 
@@ -590,7 +596,20 @@ class Connection(base.Connection):
                                                        table)
             query = query.filter(sql_condition)
 
-        return self._retrieve_samples(query, orderby, limit, table)
+        query = self._apply_options(query,
+                                    orderby,
+                                    limit,
+                                    table)
+
+        retrieve = {models.Meter: self._retrieve_samples,
+                    models.Alarm: self._retrieve_alarms}
+        return retrieve[table](query)
+
+    def query_samples(self, filter_expr=None, orderby=None, limit=None):
+        return self._retrieve_data(filter_expr,
+                                   orderby,
+                                   limit,
+                                   models.Meter)
 
     def _transform_expression(self, expression_tree, table):
 
@@ -735,6 +754,9 @@ class Connection(base.Connection):
                                 rule=row.rule,
                                 repeat_actions=row.repeat_actions)
 
+    def _retrieve_alarms(self, query):
+        return (self._row_to_alarm_model(x) for x in query.all())
+
     def get_alarms(self, name=None, user=None,
                    project=None, enabled=None, alarm_id=None, pagination=None):
         """Yields a lists of alarms that match filters
@@ -761,7 +783,7 @@ class Connection(base.Connection):
         if alarm_id is not None:
             query = query.filter(models.Alarm.id == alarm_id)
 
-        return (self._row_to_alarm_model(x) for x in query.all())
+        return self._retrieve_alarms(query)
 
     def create_alarm(self, alarm):
         """Create an alarm.
@@ -813,6 +835,11 @@ class Connection(base.Connection):
                                       on_behalf_of=row.on_behalf_of,
                                       timestamp=row.timestamp)
 
+    def query_alarms(self, filter_expr=None, orderby=None, limit=None):
+        """Yields a lists of alarms that match filter
+        """
+        return self._retrieve_data(filter_expr, orderby, limit, models.Alarm)
+
     def get_alarm_changes(self, alarm_id, on_behalf_of,
                           user=None, project=None, type=None,
                           start_timestamp=None, start_timestamp_op=None,

ceilometer/tests/api/v2/test_complex_query_scenarios.py

@@ -26,6 +26,7 @@ import testscenarios
 from ceilometer.openstack.common import timeutils
 from ceilometer.publisher import utils
 from ceilometer import sample
+from ceilometer.storage import models
 from ceilometer.tests.api import v2 as tests_api
 from ceilometer.tests import db as tests_db
 
@@ -33,6 +34,13 @@ load_tests = testscenarios.load_tests_apply_scenarios
 
 LOG = logging.getLogger(__name__)
 
+admin_header = {"X-Roles": "admin",
+                "X-Project-Id":
+                "project-id1"}
+non_admin_header = {"X-Roles": "Member",
+                    "X-Project-Id":
+                    "project-id1"}
+
 
 class TestQueryMetersController(tests_api.FunctionalTest,
                                 tests_db.MixinTestsWithBackendScenarios):
@@ -40,12 +48,7 @@ class TestQueryMetersController(tests_api.FunctionalTest,
     def setUp(self):
         super(TestQueryMetersController, self).setUp()
         self.url = '/query/samples'
-        self.admin_header = {"X-Roles": "admin",
-                             "X-Project-Id":
-                             "project-id1"}
-        self.non_admin_header = {"X-Roles": "Member",
-                                 "X-Project-Id":
-                                 "project-id1"}
+
         for cnt in [
             sample.Sample('meter.test',
                           'cumulative',
@@ -103,7 +106,7 @@ class TestQueryMetersController(tests_api.FunctionalTest,
     def test_non_admin_tenant_sees_only_its_own_project(self):
         data = self.post_json(self.url,
                               params={},
-                              headers=self.non_admin_header)
+                              headers=non_admin_header)
         for sample in data.json:
             self.assertEqual("project-id1", sample['project_id'])
 
@@ -112,7 +115,7 @@ class TestQueryMetersController(tests_api.FunctionalTest,
                               params={"filter":
                                       '{"=": {"project_id": "project-id2"}}'},
                               expect_errors=True,
-                              headers=self.non_admin_header)
+                              headers=non_admin_header)
 
         self.assertEqual(401, data.status_int)
         self.assertIn("Not Authorized to access project project-id2",
@@ -122,7 +125,7 @@ class TestQueryMetersController(tests_api.FunctionalTest,
         data = self.post_json(self.url,
                               params={"filter":
                                       '{"=": {"project_id": "project-id1"}}'},
-                              headers=self.non_admin_header)
+                              headers=non_admin_header)
 
         for sample in data.json:
             self.assertEqual("project-id1", sample['project_id'])
@@ -130,7 +133,7 @@ class TestQueryMetersController(tests_api.FunctionalTest,
     def test_admin_tenant_sees_every_project(self):
         data = self.post_json(self.url,
                               params={},
-                              headers=self.admin_header)
+                              headers=admin_header)
 
         self.assertEqual(2, len(data.json))
         for sample in data.json:
@@ -143,7 +146,7 @@ class TestQueryMetersController(tests_api.FunctionalTest,
                   '{"=": {"project_id": "project-id2"}}]}')
         data = self.post_json(self.url,
                               params={"filter": filter},
-                              headers=self.admin_header)
+                              headers=admin_header)
 
         self.assertEqual(2, len(data.json))
         for sample in data.json:
@@ -154,7 +157,7 @@ class TestQueryMetersController(tests_api.FunctionalTest,
         data = self.post_json(self.url,
                               params={"filter":
                                       '{"=": {"project_id": "project-id2"}}'},
-                              headers=self.admin_header)
+                              headers=admin_header)
 
         self.assertEqual(1, len(data.json))
         for sample in data.json:
@@ -190,3 +193,150 @@ class TestQueryMetersController(tests_api.FunctionalTest,
 
         self.assertEqual(400, data.status_int)
         self.assertIn("Limit should be positive", data.body)
+
+
+class TestQueryAlarmsController(tests_api.FunctionalTest,
+                                tests_db.MixinTestsWithBackendScenarios):
+
+    def setUp(self):
+        super(TestQueryAlarmsController, self).setUp()
+        self.alarm_url = '/query/alarms'
+
+        for state in ['ok', 'alarm', 'insufficient data']:
+            for date in [datetime.datetime(2013, 1, 1),
+                         datetime.datetime(2013, 2, 2)]:
+                for id in [1, 2]:
+                    alarm_id = "-".join([state, date.isoformat(), str(id)])
+                    project_id = "project-id%d" % id
+                    alarm = models.Alarm(name=alarm_id,
+                                         type='threshold',
+                                         enabled=True,
+                                         alarm_id=alarm_id,
+                                         description='a',
+                                         state=state,
+                                         state_timestamp=date,
+                                         timestamp=date,
+                                         ok_actions=[],
+                                         insufficient_data_actions=[],
+                                         alarm_actions=[],
+                                         repeat_actions=True,
+                                         user_id="user-id%d" % id,
+                                         project_id=project_id,
+                                         rule=dict(comparison_operator='gt',
+                                                   threshold=2.0,
+                                                   statistic='avg',
+                                                   evaluation_periods=60,
+                                                   period=1,
+                                                   meter_name='meter.test',
+                                                   query=[{'field':
+                                                           'project_id',
+                                                           'op': 'eq',
+                                                           'value':
+                                                           project_id}]))
+                    self.conn.update_alarm(alarm)
+
+    def test_query_all(self):
+        data = self.post_json(self.alarm_url,
+                              params={})
+
+        self.assertEqual(12, len(data.json))
+
+    def test_filter_with_isotime_timestamp(self):
+        date_time = datetime.datetime(2013, 1, 1)
+        isotime = date_time.isoformat()
+
+        data = self.post_json(self.alarm_url,
+                              params={"filter":
+                                      '{">": {"timestamp": "'
+                                      + isotime + '"}}'})
+
+        self.assertEqual(6, len(data.json))
+        for alarm in data.json:
+            result_time = timeutils.parse_isotime(alarm['timestamp'])
+            result_time = result_time.replace(tzinfo=None)
+            self.assertTrue(result_time > date_time)
+
+    def test_filter_with_isotime_state_timestamp(self):
+        date_time = datetime.datetime(2013, 1, 1)
+        isotime = date_time.isoformat()
+
+        data = self.post_json(self.alarm_url,
+                              params={"filter":
+                                      '{">": {"state_timestamp": "'
+                                      + isotime + '"}}'})
+
+        self.assertEqual(6, len(data.json))
+        for alarm in data.json:
+            result_time = timeutils.parse_isotime(alarm['state_timestamp'])
+            result_time = result_time.replace(tzinfo=None)
+            self.assertTrue(result_time > date_time)
+
+    def test_non_admin_tenant_sees_only_its_own_project(self):
+        data = self.post_json(self.alarm_url,
+                              params={},
+                              headers=non_admin_header)
+        for alarm in data.json:
+            self.assertEqual("project-id1", alarm['project_id'])
+
+    def test_non_admin_tenant_cannot_query_others_project(self):
+        data = self.post_json(self.alarm_url,
+                              params={"filter":
+                                      '{"=": {"project_id": "project-id2"}}'},
+                              expect_errors=True,
+                              headers=non_admin_header)
+
+        self.assertEqual(401, data.status_int)
+        self.assertIn("Not Authorized to access project project-id2",
+                      data.body)
+
+    def test_non_admin_tenant_can_explicitly_filter_for_own_project(self):
+        data = self.post_json(self.alarm_url,
+                              params={"filter":
+                                      '{"=": {"project_id": "project-id1"}}'},
+                              headers=non_admin_header)
+
+        for alarm in data.json:
+            self.assertEqual("project-id1", alarm['project_id'])
+
+    def test_admin_tenant_sees_every_project(self):
+        data = self.post_json(self.alarm_url,
+                              params={},
+                              headers=admin_header)
+
+        self.assertEqual(12, len(data.json))
+        for alarm in data.json:
+            self.assertIn(alarm['project_id'],
+                          (["project-id1", "project-id2"]))
+
+    def test_admin_tenant_can_query_any_project(self):
+        data = self.post_json(self.alarm_url,
+                              params={"filter":
+                                      '{"=": {"project_id": "project-id2"}}'},
+                              headers=admin_header)
+
+        self.assertEqual(6, len(data.json))
+        for alarm in data.json:
+            self.assertIn(alarm['project_id'], set(["project-id2"]))
+
+    def test_query_with_filter_orderby_and_limit(self):
+        orderby = '[{"state_timestamp": "DESC"}]'
+        data = self.post_json(self.alarm_url,
+                              params={"filter": '{"=": {"state": "alarm"}}',
+                                      "orderby": orderby,
+                                      "limit": 3})
+
+        self.assertEqual(3, len(data.json))
+        self.assertEqual(["2013-02-02T00:00:00",
+                          "2013-02-02T00:00:00",
+                          "2013-01-01T00:00:00"],
+                         [a["state_timestamp"] for a in data.json])
+        for alarm in data.json:
+            self.assertEqual("alarm", alarm["state"])
+
+    def test_limit_should_be_positive(self):
+        data = self.post_json(self.alarm_url,
+                              params={"limit": 0},
+                              expect_errors=True)
+
+        self.assertEqual(400, data.status_int)
+        self.assertIn("Limit should be positive", data.body)

ceilometer/tests/storage/test_storage_scenarios.py

@@ -2290,6 +2290,47 @@ class AlarmTestPagination(AlarmTestBase,
                          [i.name for i in page1])
 
 
+class ComplexAlarmQueryTest(AlarmTestBase,
+                            tests_db.MixinTestsWithBackendScenarios):
+
+    def test_no_filter(self):
+        self.add_some_alarms()
+        result = list(self.conn.query_alarms())
+        self.assertEqual(3, len(result))
+
+    def test_no_filter_with_limit(self):
+        self.add_some_alarms()
+        result = list(self.conn.query_alarms(limit=2))
+        self.assertEqual(2, len(result))
+
+    def test_filter(self):
+        self.add_some_alarms()
+        filter_expr = {"and":
+                       [{"or":
+                        [{"=": {"name": "yellow-alert"}},
+                         {"=": {"name": "red-alert"}}]},
+                       {"=": {"enabled": True}}]}
+
+        result = list(self.conn.query_alarms(filter_expr=filter_expr))
+
+        self.assertEqual(1, len(result))
+        for a in result:
+            self.assertIn(a.name, set(["yellow-alert", "red-alert"]))
+            self.assertTrue(a.enabled)
+
+    def test_filter_and_orderby(self):
+        self.add_some_alarms()
+        result = list(self.conn.query_alarms(filter_expr={"=":
+                                                          {"enabled":
+                                                          True}},
+                                             orderby=[{"name": "asc"}]))
+        self.assertEqual(2, len(result))
+        self.assertEqual(["orange-alert", "red-alert"],
+                         [a.name for a in result])
+        for a in result:
+            self.assertTrue(a.enabled)
+
+
 class EventTestBase(tests_db.TestBase,
                     tests_db.MixinTestsWithBackendScenarios):
     """Separate test base class because we don't want to

doc/source/webapi/v2.rst

@@ -94,8 +94,9 @@ field of *Sample*).
 Complex Query
 +++++++++++++
 The filter expressions of the Complex Query feature operate on the fields
-of *Sample*. The following comparison operators are supported: *=*, *!=*, *<*,
-*<=*, *>* and *>=*; and the following logical operators can be used: *and* and *or*.
+of *Sample* and *Alarm*. The following comparison operators are supported: *=*,
+*!=*, *<*, *<=*, *>* and *>=*; and the following logical operators can be
+used: *and* and *or*.
 
 Complex Query supports defining the list of orderby expressions in the form
 of [{"field_name": "asc"}, {"field_name2": "desc"}, ...].
@@ -107,6 +108,9 @@ The *filter*, *orderby* and *limit* are all optional fields in a query.
 .. rest-controller:: ceilometer.api.controllers.v2:QuerySamplesController
    :webprefix: /v2/query/samples
 
+.. rest-controller:: ceilometer.api.controllers.v2:QueryAlarmsController
+   :webprefix: /v2/query/alarms
+
 .. autotype:: ceilometer.api.controllers.v2.ComplexQuery
    :members: