From ff83a9d1ea231ad4648e0f8b10068497a983f503 Mon Sep 17 00:00:00 2001 From: Ildiko Vancsa Date: Sat, 14 Dec 2013 12:44:24 +0100 Subject: [PATCH] Implements complex query functionality for alarms New API resource /query/alarms has been added Implements: blueprint complex-filter-expressions-in-api-queries Change-Id: I3289f187c70a74d19381c24a20d324c2d14a19fb --- ceilometer/api/controllers/v2.py | 25 ++- ceilometer/storage/base.py | 12 ++ ceilometer/storage/impl_mongodb.py | 37 +++- ceilometer/storage/impl_sqlalchemy.py | 39 +++- .../api/v2/test_complex_query_scenarios.py | 174 ++++++++++++++++-- .../tests/storage/test_storage_scenarios.py | 41 +++++ doc/source/webapi/v2.rst | 8 +- 7 files changed, 307 insertions(+), 29 deletions(-) diff --git a/ceilometer/api/controllers/v2.py b/ceilometer/api/controllers/v2.py index 76dbd318d..93db1f040 100644 --- a/ceilometer/api/controllers/v2.py +++ b/ceilometer/api/controllers/v2.py @@ -3,11 +3,14 @@ # Copyright © 2012 New Dream Network, LLC (DreamHost) # Copyright 2013 IBM Corp. # Copyright © 2013 eNovance +# Copyright Ericsson AB 2013. All rights reserved # # Authors: Doug Hellmann # Angus Salkeld # Eoghan Glynn # Julien Danjou +# Ildiko Vancsa +# Balazs Gibizer # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -1058,7 +1061,7 @@ class ValidatedComplexQuery(object): "minProperties": 1, "maxProperties": 1}} - timestamp_fields = ["timestamp"] + timestamp_fields = ["timestamp", "state_timestamp"] def __init__(self, query): self.original_query = query @@ -2049,8 +2052,28 @@ class QuerySamplesController(rest.RestController): query.limit)] +class QueryAlarmsController(rest.RestController): + """Provides complex query possibilities for alarms + """ + @wsme_pecan.wsexpose([Alarm], body=ComplexQuery) + def post(self, body): + """Define query for retrieving Alarm data. + + :param body: Query rules for the alarms to be returned. + """ + query = ValidatedComplexQuery(body) + query.validate(visibility_field="project_id") + conn = pecan.request.storage_conn + return [Alarm.from_db_model(s) + for s in conn.query_alarms(query.filter_expr, + query.orderby, + query.limit)] + + class QueryController(rest.RestController): + samples = QuerySamplesController() + alarms = QueryAlarmsController() class V2Controller(object): diff --git a/ceilometer/storage/base.py b/ceilometer/storage/base.py index d0b6e2845..f4eb2a6d9 100644 --- a/ceilometer/storage/base.py +++ b/ceilometer/storage/base.py @@ -331,3 +331,15 @@ class Connection(object): raise NotImplementedError(_('Complex query for samples \ is not implemented.')) + + @staticmethod + def query_alarms(filter_expr=None, orderby=None, limit=None): + """Return an iterable of model.Alarm objects. + + :param filter_expr: Filter expression for query. + :param orderby: List of field name and direction pairs for order by. + :param limit: Maximum number of results to return. + """ + + raise NotImplementedError(_('Complex query for alarms \ + is not implemented.')) diff --git a/ceilometer/storage/impl_mongodb.py b/ceilometer/storage/impl_mongodb.py index 624157410..da84ca86e 100644 --- a/ceilometer/storage/impl_mongodb.py +++ b/ceilometer/storage/impl_mongodb.py @@ -779,7 +779,7 @@ class Connection(base.Connection): [("timestamp", pymongo.DESCENDING)], limit) - def query_samples(self, filter_expr=None, orderby=None, limit=None): + def _retrieve_data(self, filter_expr, orderby, limit, model): if limit == 0: return [] query_filter = {} @@ -790,7 +790,12 @@ class Connection(base.Connection): query_filter = self._transform_filter( filter_expr) - return self._retrieve_samples(query_filter, orderby_filter, limit) + retrieve = {models.Meter: self._retrieve_samples, + models.Alarm: self._retrieve_alarms} + return retrieve[model](query_filter, orderby_filter, limit) + + def query_samples(self, filter_expr=None, orderby=None, limit=None): + return self._retrieve_data(filter_expr, orderby, limit, models.Meter) def _transform_orderby(self, orderby): orderby_filter = [] @@ -944,6 +949,22 @@ class Connection(base.Connection): del alarm['matching_metadata'] alarm['rule']['query'] = query + def _retrieve_alarms(self, query_filter, orderby, limit): + if limit is not None: + alarms = self.db.alarm.find(query_filter, + limit=limit, + sort=orderby) + else: + alarms = self.db.alarm.find( + query_filter, sort=orderby) + + for alarm in alarms: + a = {} + a.update(alarm) + del a['_id'] + self._ensure_encapsulated_rule_format(a) + yield models.Alarm(**a) + def get_alarms(self, name=None, user=None, project=None, enabled=None, alarm_id=None, pagination=None): """Yields a lists of alarms that match filters @@ -969,12 +990,7 @@ class Connection(base.Connection): if alarm_id is not None: q['alarm_id'] = alarm_id - for alarm in self.db.alarm.find(q): - a = {} - a.update(alarm) - del a['_id'] - self._ensure_encapsulated_rule_format(a) - yield models.Alarm(**a) + return self._retrieve_alarms(q, [], None) def update_alarm(self, alarm): """update alarm @@ -1052,3 +1068,8 @@ class Connection(base.Connection): """Record alarm change event. """ self.db.alarm_history.insert(alarm_change) + + def query_alarms(self, filter_expr=None, orderby=None, limit=None): + """Return an iterable of model.Alarm objects. + """ + return self._retrieve_data(filter_expr, orderby, limit, models.Alarm) diff --git a/ceilometer/storage/impl_sqlalchemy.py b/ceilometer/storage/impl_sqlalchemy.py index 29074fe22..1deab2cc4 100644 --- a/ceilometer/storage/impl_sqlalchemy.py +++ b/ceilometer/storage/impl_sqlalchemy.py @@ -532,10 +532,13 @@ class Connection(base.Connection): source=resource.sources[0].id, user_id=resource.user_id) - def _retrieve_samples(self, query, orderby, limit, table): + def _apply_options(self, query, orderby, limit, table): query = self._apply_order_by(query, orderby, table) if limit is not None: query = query.limit(limit) + return query + + def _retrieve_samples(self, query): samples = query.all() for s in samples: @@ -575,13 +578,16 @@ class Connection(base.Connection): query = make_query_from_filter(session, query, sample_filter, require_meter=False) - return self._retrieve_samples(query, None, limit, table) + query = self._apply_options(query, + None, + limit, + table) + return self._retrieve_samples(query) - def query_samples(self, filter_expr=None, orderby=None, limit=None): + def _retrieve_data(self, filter_expr, orderby, limit, table): if limit == 0: return [] - table = models.Meter session = self._get_db_session() query = session.query(table) @@ -590,7 +596,20 @@ class Connection(base.Connection): table) query = query.filter(sql_condition) - return self._retrieve_samples(query, orderby, limit, table) + query = self._apply_options(query, + orderby, + limit, + table) + + retrieve = {models.Meter: self._retrieve_samples, + models.Alarm: self._retrieve_alarms} + return retrieve[table](query) + + def query_samples(self, filter_expr=None, orderby=None, limit=None): + return self._retrieve_data(filter_expr, + orderby, + limit, + models.Meter) def _transform_expression(self, expression_tree, table): @@ -735,6 +754,9 @@ class Connection(base.Connection): rule=row.rule, repeat_actions=row.repeat_actions) + def _retrieve_alarms(self, query): + return (self._row_to_alarm_model(x) for x in query.all()) + def get_alarms(self, name=None, user=None, project=None, enabled=None, alarm_id=None, pagination=None): """Yields a lists of alarms that match filters @@ -761,7 +783,7 @@ class Connection(base.Connection): if alarm_id is not None: query = query.filter(models.Alarm.id == alarm_id) - return (self._row_to_alarm_model(x) for x in query.all()) + return self._retrieve_alarms(query) def create_alarm(self, alarm): """Create an alarm. @@ -813,6 +835,11 @@ class Connection(base.Connection): on_behalf_of=row.on_behalf_of, timestamp=row.timestamp) + def query_alarms(self, filter_expr=None, orderby=None, limit=None): + """Yields a lists of alarms that match filter + """ + return self._retrieve_data(filter_expr, orderby, limit, models.Alarm) + def get_alarm_changes(self, alarm_id, on_behalf_of, user=None, project=None, type=None, start_timestamp=None, start_timestamp_op=None, diff --git a/ceilometer/tests/api/v2/test_complex_query_scenarios.py b/ceilometer/tests/api/v2/test_complex_query_scenarios.py index db530f54c..3dfb9e24c 100644 --- a/ceilometer/tests/api/v2/test_complex_query_scenarios.py +++ b/ceilometer/tests/api/v2/test_complex_query_scenarios.py @@ -26,6 +26,7 @@ import testscenarios from ceilometer.openstack.common import timeutils from ceilometer.publisher import utils from ceilometer import sample +from ceilometer.storage import models from ceilometer.tests.api import v2 as tests_api from ceilometer.tests import db as tests_db @@ -33,6 +34,13 @@ load_tests = testscenarios.load_tests_apply_scenarios LOG = logging.getLogger(__name__) +admin_header = {"X-Roles": "admin", + "X-Project-Id": + "project-id1"} +non_admin_header = {"X-Roles": "Member", + "X-Project-Id": + "project-id1"} + class TestQueryMetersController(tests_api.FunctionalTest, tests_db.MixinTestsWithBackendScenarios): @@ -40,12 +48,7 @@ class TestQueryMetersController(tests_api.FunctionalTest, def setUp(self): super(TestQueryMetersController, self).setUp() self.url = '/query/samples' - self.admin_header = {"X-Roles": "admin", - "X-Project-Id": - "project-id1"} - self.non_admin_header = {"X-Roles": "Member", - "X-Project-Id": - "project-id1"} + for cnt in [ sample.Sample('meter.test', 'cumulative', @@ -103,7 +106,7 @@ class TestQueryMetersController(tests_api.FunctionalTest, def test_non_admin_tenant_sees_only_its_own_project(self): data = self.post_json(self.url, params={}, - headers=self.non_admin_header) + headers=non_admin_header) for sample in data.json: self.assertEqual("project-id1", sample['project_id']) @@ -112,7 +115,7 @@ class TestQueryMetersController(tests_api.FunctionalTest, params={"filter": '{"=": {"project_id": "project-id2"}}'}, expect_errors=True, - headers=self.non_admin_header) + headers=non_admin_header) self.assertEqual(401, data.status_int) self.assertIn("Not Authorized to access project project-id2", @@ -122,7 +125,7 @@ class TestQueryMetersController(tests_api.FunctionalTest, data = self.post_json(self.url, params={"filter": '{"=": {"project_id": "project-id1"}}'}, - headers=self.non_admin_header) + headers=non_admin_header) for sample in data.json: self.assertEqual("project-id1", sample['project_id']) @@ -130,7 +133,7 @@ class TestQueryMetersController(tests_api.FunctionalTest, def test_admin_tenant_sees_every_project(self): data = self.post_json(self.url, params={}, - headers=self.admin_header) + headers=admin_header) self.assertEqual(2, len(data.json)) for sample in data.json: @@ -143,7 +146,7 @@ class TestQueryMetersController(tests_api.FunctionalTest, '{"=": {"project_id": "project-id2"}}]}') data = self.post_json(self.url, params={"filter": filter}, - headers=self.admin_header) + headers=admin_header) self.assertEqual(2, len(data.json)) for sample in data.json: @@ -154,7 +157,7 @@ class TestQueryMetersController(tests_api.FunctionalTest, data = self.post_json(self.url, params={"filter": '{"=": {"project_id": "project-id2"}}'}, - headers=self.admin_header) + headers=admin_header) self.assertEqual(1, len(data.json)) for sample in data.json: @@ -190,3 +193,150 @@ class TestQueryMetersController(tests_api.FunctionalTest, self.assertEqual(400, data.status_int) self.assertIn("Limit should be positive", data.body) + + +class TestQueryAlarmsController(tests_api.FunctionalTest, + tests_db.MixinTestsWithBackendScenarios): + + def setUp(self): + super(TestQueryAlarmsController, self).setUp() + self.alarm_url = '/query/alarms' + + for state in ['ok', 'alarm', 'insufficient data']: + for date in [datetime.datetime(2013, 1, 1), + datetime.datetime(2013, 2, 2)]: + for id in [1, 2]: + alarm_id = "-".join([state, date.isoformat(), str(id)]) + project_id = "project-id%d" % id + alarm = models.Alarm(name=alarm_id, + type='threshold', + enabled=True, + alarm_id=alarm_id, + description='a', + state=state, + state_timestamp=date, + timestamp=date, + ok_actions=[], + insufficient_data_actions=[], + alarm_actions=[], + repeat_actions=True, + user_id="user-id%d" % id, + project_id=project_id, + rule=dict(comparison_operator='gt', + threshold=2.0, + statistic='avg', + evaluation_periods=60, + period=1, + meter_name='meter.test', + query=[{'field': + 'project_id', + 'op': 'eq', + 'value': + project_id}])) + self.conn.update_alarm(alarm) + + def test_query_all(self): + data = self.post_json(self.alarm_url, + params={}) + + self.assertEqual(12, len(data.json)) + + def test_filter_with_isotime_timestamp(self): + date_time = datetime.datetime(2013, 1, 1) + isotime = date_time.isoformat() + + data = self.post_json(self.alarm_url, + params={"filter": + '{">": {"timestamp": "' + + isotime + '"}}'}) + + self.assertEqual(6, len(data.json)) + for alarm in data.json: + result_time = timeutils.parse_isotime(alarm['timestamp']) + result_time = result_time.replace(tzinfo=None) + self.assertTrue(result_time > date_time) + + def test_filter_with_isotime_state_timestamp(self): + date_time = datetime.datetime(2013, 1, 1) + isotime = date_time.isoformat() + + data = self.post_json(self.alarm_url, + params={"filter": + '{">": {"state_timestamp": "' + + isotime + '"}}'}) + + self.assertEqual(6, len(data.json)) + for alarm in data.json: + result_time = timeutils.parse_isotime(alarm['state_timestamp']) + result_time = result_time.replace(tzinfo=None) + self.assertTrue(result_time > date_time) + + def test_non_admin_tenant_sees_only_its_own_project(self): + data = self.post_json(self.alarm_url, + params={}, + headers=non_admin_header) + for alarm in data.json: + self.assertEqual("project-id1", alarm['project_id']) + + def test_non_admin_tenant_cannot_query_others_project(self): + data = self.post_json(self.alarm_url, + params={"filter": + '{"=": {"project_id": "project-id2"}}'}, + expect_errors=True, + headers=non_admin_header) + + self.assertEqual(401, data.status_int) + self.assertIn("Not Authorized to access project project-id2", + data.body) + + def test_non_admin_tenant_can_explicitly_filter_for_own_project(self): + data = self.post_json(self.alarm_url, + params={"filter": + '{"=": {"project_id": "project-id1"}}'}, + headers=non_admin_header) + + for alarm in data.json: + self.assertEqual("project-id1", alarm['project_id']) + + def test_admin_tenant_sees_every_project(self): + data = self.post_json(self.alarm_url, + params={}, + headers=admin_header) + + self.assertEqual(12, len(data.json)) + for alarm in data.json: + self.assertIn(alarm['project_id'], + (["project-id1", "project-id2"])) + + def test_admin_tenant_can_query_any_project(self): + data = self.post_json(self.alarm_url, + params={"filter": + '{"=": {"project_id": "project-id2"}}'}, + headers=admin_header) + + self.assertEqual(6, len(data.json)) + for alarm in data.json: + self.assertIn(alarm['project_id'], set(["project-id2"])) + + def test_query_with_filter_orderby_and_limit(self): + orderby = '[{"state_timestamp": "DESC"}]' + data = self.post_json(self.alarm_url, + params={"filter": '{"=": {"state": "alarm"}}', + "orderby": orderby, + "limit": 3}) + + self.assertEqual(3, len(data.json)) + self.assertEqual(["2013-02-02T00:00:00", + "2013-02-02T00:00:00", + "2013-01-01T00:00:00"], + [a["state_timestamp"] for a in data.json]) + for alarm in data.json: + self.assertEqual("alarm", alarm["state"]) + + def test_limit_should_be_positive(self): + data = self.post_json(self.alarm_url, + params={"limit": 0}, + expect_errors=True) + + self.assertEqual(400, data.status_int) + self.assertIn("Limit should be positive", data.body) diff --git a/ceilometer/tests/storage/test_storage_scenarios.py b/ceilometer/tests/storage/test_storage_scenarios.py index acfc2bf76..c07ac6ff2 100644 --- a/ceilometer/tests/storage/test_storage_scenarios.py +++ b/ceilometer/tests/storage/test_storage_scenarios.py @@ -2290,6 +2290,47 @@ class AlarmTestPagination(AlarmTestBase, [i.name for i in page1]) +class ComplexAlarmQueryTest(AlarmTestBase, + tests_db.MixinTestsWithBackendScenarios): + + def test_no_filter(self): + self.add_some_alarms() + result = list(self.conn.query_alarms()) + self.assertEqual(3, len(result)) + + def test_no_filter_with_limit(self): + self.add_some_alarms() + result = list(self.conn.query_alarms(limit=2)) + self.assertEqual(2, len(result)) + + def test_filter(self): + self.add_some_alarms() + filter_expr = {"and": + [{"or": + [{"=": {"name": "yellow-alert"}}, + {"=": {"name": "red-alert"}}]}, + {"=": {"enabled": True}}]} + + result = list(self.conn.query_alarms(filter_expr=filter_expr)) + + self.assertEqual(1, len(result)) + for a in result: + self.assertIn(a.name, set(["yellow-alert", "red-alert"])) + self.assertTrue(a.enabled) + + def test_filter_and_orderby(self): + self.add_some_alarms() + result = list(self.conn.query_alarms(filter_expr={"=": + {"enabled": + True}}, + orderby=[{"name": "asc"}])) + self.assertEqual(2, len(result)) + self.assertEqual(["orange-alert", "red-alert"], + [a.name for a in result]) + for a in result: + self.assertTrue(a.enabled) + + class EventTestBase(tests_db.TestBase, tests_db.MixinTestsWithBackendScenarios): """Separate test base class because we don't want to diff --git a/doc/source/webapi/v2.rst b/doc/source/webapi/v2.rst index b34b8860e..06e9bfeb8 100644 --- a/doc/source/webapi/v2.rst +++ b/doc/source/webapi/v2.rst @@ -94,8 +94,9 @@ field of *Sample*). Complex Query +++++++++++++ The filter expressions of the Complex Query feature operate on the fields -of *Sample*. The following comparison operators are supported: *=*, *!=*, *<*, -*<=*, *>* and *>=*; and the following logical operators can be used: *and* and *or*. +of *Sample* and *Alarm*. The following comparison operators are supported: *=*, +*!=*, *<*, *<=*, *>* and *>=*; and the following logical operators can be +used: *and* and *or*. Complex Query supports defining the list of orderby expressions in the form of [{"field_name": "asc"}, {"field_name2": "desc"}, ...]. @@ -107,6 +108,9 @@ The *filter*, *orderby* and *limit* are all optional fields in a query. .. rest-controller:: ceilometer.api.controllers.v2:QuerySamplesController :webprefix: /v2/query/samples +.. rest-controller:: ceilometer.api.controllers.v2:QueryAlarmsController + :webprefix: /v2/query/alarms + .. autotype:: ceilometer.api.controllers.v2.ComplexQuery :members: