From 4d57208adda0696e5163285b5efd397ef3324231 Mon Sep 17 00:00:00 2001 From: Ildiko Vancsa Date: Sun, 16 Mar 2014 10:32:11 +0100 Subject: [PATCH] Fix bug in get_capabilities behavior in DB drivers Capabilities API returns NotImplementedError in case of SQLAlchemy driver. This issue is fixed in this patch by moving the function into the proper class. Another issue is, that get_capabilities function overwrites the DEFAULT_CAPABILITIES dict in base.py every time, when the function is invoked. This behavior was changed to create a CAPABILITIES dict in each DB driver's __init__ function by making a deep copy from DEFAULT_CAPABILITIES and updating the new dict with the AVAILABLE_CAPABILITIES. get_capabilities now returns the newly created CAPABILITIES dict without modifying it. Tests were also added to check that get_capabilities returns the expected values for each DB driver. Fixes-bug: #1292611 Change-Id: I725751b600bf462c19278e5785eb2d8530023083 --- ceilometer/storage/impl_db2.py | 39 +++++----- ceilometer/storage/impl_hbase.py | 29 ++++--- ceilometer/storage/impl_mongodb.py | 59 +++++++------- ceilometer/storage/impl_sqlalchemy.py | 71 ++++++++--------- ceilometer/tests/api/v2/test_capabilities.py | 37 +++++++++ ceilometer/tests/storage/test_impl_db2.py | 76 +++++++++++++++++++ ceilometer/tests/storage/test_impl_hbase.py | 45 +++++++++++ ceilometer/tests/storage/test_impl_mongodb.py | 45 +++++++++++ .../tests/storage/test_impl_sqlalchemy.py | 45 +++++++++++ ceilometer/utils.py | 16 ++-- 10 files changed, 365 insertions(+), 97 deletions(-) create mode 100644 ceilometer/tests/api/v2/test_capabilities.py create mode 100644 ceilometer/tests/storage/test_impl_db2.py diff --git a/ceilometer/storage/impl_db2.py b/ceilometer/storage/impl_db2.py index abef6f2fe..a2805dfea 100644 --- a/ceilometer/storage/impl_db2.py +++ b/ceilometer/storage/impl_db2.py @@ -75,6 +75,24 @@ class DB2Storage(base.StorageEngine): return Connection(conf) +AVAILABLE_CAPABILITIES = { + 'meters': {'query': {'simple': True, + 'metadata': True}}, + 'resources': {'query': {'simple': True, + 'metadata': True}}, + 'samples': {'query': {'simple': True, + 'metadata': True, + 'complex': True}}, + 'statistics': {'groupby': True, + 'query': {'simple': True, + 'metadata': True}, + 'aggregation': {'standard': True}}, + 'alarms': {'query': {'simple': True, + 'complex': True}, + 'history': {'query': {'simple': True}}}, +} + + class Connection(pymongo_base.Connection): """DB2 connection. """ @@ -132,6 +150,9 @@ class Connection(pymongo_base.Connection): self.db.authenticate(connection_options['username'], connection_options['password']) + self.CAPABILITIES = utils.update_nested(self.DEFAULT_CAPABILITIES, + AVAILABLE_CAPABILITIES) + self.upgrade() @classmethod @@ -423,20 +444,4 @@ class Connection(pymongo_base.Connection): def get_capabilities(self): """Return an dictionary representing the capabilities of this driver. """ - available = { - 'meters': {'query': {'simple': True, - 'metadata': True}}, - 'resources': {'query': {'simple': True, - 'metadata': True}}, - 'samples': {'query': {'simple': True, - 'metadata': True, - 'complex': True}}, - 'statistics': {'groupby': True, - 'query': {'simple': True, - 'metadata': True}, - 'aggregation': {'standard': True}}, - 'alarms': {'query': {'simple': True, - 'complex': True}, - 'history': {'query': {'simple': True}}}, - } - return utils.update_nested(self.DEFAULT_CAPABILITIES, available) + return self.CAPABILITIES diff --git a/ceilometer/storage/impl_hbase.py b/ceilometer/storage/impl_hbase.py index 80565deef..2a8677d0d 100644 --- a/ceilometer/storage/impl_hbase.py +++ b/ceilometer/storage/impl_hbase.py @@ -94,6 +94,19 @@ class HBaseStorage(base.StorageEngine): return Connection(conf) +AVAILABLE_CAPABILITIES = { + 'meters': {'query': {'simple': True, + 'metadata': True}}, + 'resources': {'query': {'simple': True, + 'metadata': True}}, + 'samples': {'query': {'simple': True, + 'metadata': True}}, + 'statistics': {'query': {'simple': True, + 'metadata': True}, + 'aggregation': {'standard': True}}, +} + + class Connection(base.Connection): """HBase connection. """ @@ -128,6 +141,9 @@ class Connection(base.Connection): self.conn = self._get_connection(opts) self.conn.open() + self.CAPABILITIES = utils.update_nested(self.DEFAULT_CAPABILITIES, + AVAILABLE_CAPABILITIES) + def upgrade(self): self.conn.create_table(self.PROJECT_TABLE, {'f': dict()}) self.conn.create_table(self.USER_TABLE, {'f': dict()}) @@ -565,18 +581,7 @@ class Connection(base.Connection): def get_capabilities(self): """Return an dictionary representing the capabilities of this driver. """ - available = { - 'meters': {'query': {'simple': True, - 'metadata': True}}, - 'resources': {'query': {'simple': True, - 'metadata': True}}, - 'samples': {'query': {'simple': True, - 'metadata': True}}, - 'statistics': {'query': {'simple': True, - 'metadata': True}, - 'aggregation': {'standard': True}}, - } - return utils.update_nested(self.DEFAULT_CAPABILITIES, available) + return self.CAPABILITIES ############### diff --git a/ceilometer/storage/impl_mongodb.py b/ceilometer/storage/impl_mongodb.py index ed984d0a2..991b57149 100644 --- a/ceilometer/storage/impl_mongodb.py +++ b/ceilometer/storage/impl_mongodb.py @@ -81,6 +81,34 @@ class MongoDBStorage(base.StorageEngine): return Connection(conf) +AVAILABLE_CAPABILITIES = { + 'meters': {'query': {'simple': True, + 'metadata': True}}, + 'resources': {'query': {'simple': True, + 'metadata': True}}, + 'samples': {'query': {'simple': True, + 'metadata': True, + 'complex': True}}, + 'statistics': {'groupby': True, + 'query': {'simple': True, + 'metadata': True}, + 'aggregation': {'standard': True, + 'selectable': { + 'max': True, + 'min': True, + 'sum': True, + 'avg': True, + 'count': True, + 'stddev': True, + 'cardinality': True}} + }, + 'alarms': {'query': {'simple': True, + 'complex': True}, + 'history': {'query': {'simple': True, + 'complex': True}}}, +} + + class Connection(pymongo_base.Connection): """MongoDB connection. """ @@ -404,6 +432,9 @@ class Connection(pymongo_base.Connection): self.db.authenticate(connection_options['username'], connection_options['password']) + self.CAPABILITIES = utils.update_nested(self.DEFAULT_CAPABILITIES, + AVAILABLE_CAPABILITIES) + # NOTE(jd) Upgrading is just about creating index, so let's do this # on connection to be sure at least the TTL is correcly updated if # needed. @@ -859,30 +890,4 @@ class Connection(pymongo_base.Connection): def get_capabilities(self): """Return an dictionary representing the capabilities of this driver. """ - available = { - 'meters': {'query': {'simple': True, - 'metadata': True}}, - 'resources': {'query': {'simple': True, - 'metadata': True}}, - 'samples': {'query': {'simple': True, - 'metadata': True, - 'complex': True}}, - 'statistics': {'groupby': True, - 'query': {'simple': True, - 'metadata': True}, - 'aggregation': {'standard': True, - 'selectable': { - 'max': True, - 'min': True, - 'sum': True, - 'avg': True, - 'count': True, - 'stddev': True, - 'cardinality': True}} - }, - 'alarms': {'query': {'simple': True, - 'complex': True}, - 'history': {'query': {'simple': True, - 'complex': True}}}, - } - return utils.update_nested(self.DEFAULT_CAPABILITIES, available) + return self.CAPABILITIES diff --git a/ceilometer/storage/impl_sqlalchemy.py b/ceilometer/storage/impl_sqlalchemy.py index 886bc8e78..75f6e3f7b 100644 --- a/ceilometer/storage/impl_sqlalchemy.py +++ b/ceilometer/storage/impl_sqlalchemy.py @@ -133,6 +133,36 @@ PARAMETERIZED_AGGREGATES = dict( ) ) +AVAILABLE_CAPABILITIES = { + 'meters': {'query': {'simple': True, + 'metadata': True}}, + 'resources': {'query': {'simple': True, + 'metadata': True}}, + 'samples': {'pagination': True, + 'groupby': True, + 'query': {'simple': True, + 'metadata': True, + 'complex': True}}, + 'statistics': {'groupby': True, + 'query': {'simple': True, + 'metadata': True}, + 'aggregation': {'standard': True, + 'selectable': { + 'max': True, + 'min': True, + 'sum': True, + 'avg': True, + 'count': True, + 'stddev': True, + 'cardinality': True}} + }, + 'alarms': {'query': {'simple': True, + 'complex': True}, + 'history': {'query': {'simple': True, + 'complex': True}}}, + 'events': {'query': {'simple': True}}, +} + def apply_metaquery_filter(session, query, metaquery): """Apply provided metaquery filter to existing query. @@ -223,6 +253,8 @@ class Connection(base.Connection): self._maker = sqlalchemy_session.get_maker(self._engine) sqlalchemy_session._ENGINE = None sqlalchemy_session._MAKER = None + self._CAPABILITIES = utils.update_nested(self.DEFAULT_CAPABILITIES, + AVAILABLE_CAPABILITIES) def _get_db_session(self): return self._maker() @@ -1262,6 +1294,11 @@ class Connection(base.Connection): dtype=type.data_type, value=trait.get_value()) + def get_capabilities(self): + """Return an dictionary representing the capabilities of this driver. + """ + return self._CAPABILITIES + class QueryTransformer(object): operators = {"=": operator.eq, @@ -1349,37 +1386,3 @@ class QueryTransformer(object): def get_query(self): return self.query - - def get_capabilities(self): - """Return an dictionary representing the capabilities of this driver. - """ - available = { - 'meters': {'query': {'simple': True, - 'metadata': True}}, - 'resources': {'query': {'simple': True, - 'metadata': True}}, - 'samples': {'pagination': True, - 'groupby': True, - 'query': {'simple': True, - 'metadata': True, - 'complex': True}}, - 'statistics': {'groupby': True, - 'query': {'simple': True, - 'metadata': True}, - 'aggregation': {'standard': True, - 'selectable': { - 'max': True, - 'min': True, - 'sum': True, - 'avg': True, - 'count': True, - 'stddev': True, - 'cardinality': True}} - }, - 'alarms': {'query': {'simple': True, - 'complex': True}, - 'history': {'query': {'simple': True, - 'complex': True}}}, - 'events': {'query': {'simple': True}}, - } - return utils.update_nested(self.DEFAULT_CAPABILITIES, available) diff --git a/ceilometer/tests/api/v2/test_capabilities.py b/ceilometer/tests/api/v2/test_capabilities.py new file mode 100644 index 000000000..d8e6338cd --- /dev/null +++ b/ceilometer/tests/api/v2/test_capabilities.py @@ -0,0 +1,37 @@ +# -*- encoding: utf-8 -*- +# +# Copyright Ericsson AB 2014. All rights reserved +# +# Authors: Ildiko Vancsa +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import testscenarios + +from ceilometer.tests.api import v2 as tests_api +from ceilometer.tests import db as tests_db + +load_tests = testscenarios.load_tests_apply_scenarios + + +class TestCapabilitiesController(tests_api.FunctionalTest, + tests_db.MixinTestsWithBackendScenarios): + + def setUp(self): + super(TestCapabilitiesController, self).setUp() + self.url = '/capabilities' + + def test_capabilities(self): + data = self.get_json(self.url) + self.assertIsNotNone(data) + self.assertNotEqual({}, data) diff --git a/ceilometer/tests/storage/test_impl_db2.py b/ceilometer/tests/storage/test_impl_db2.py new file mode 100644 index 000000000..c40926694 --- /dev/null +++ b/ceilometer/tests/storage/test_impl_db2.py @@ -0,0 +1,76 @@ +# -*- encoding: utf-8 -*- +# +# Copyright Ericsson AB 2014. All rights reserved +# +# Authors: Ildiko Vancsa +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +"""Tests for ceilometer/storage/impl_db2.py + +.. note:: + In order to run the tests against another MongoDB server set the + environment variable CEILOMETER_TEST_DB2_URL to point to a DB2 + server before running the tests. + +""" + +from ceilometer.tests import db as tests_db + + +class DB2EngineTestBase(tests_db.TestBase): + database_connection = tests_db.DB2FakeConnectionUrl() + + +class CapabilitiesTest(DB2EngineTestBase): + # Check the returned capabilities list, which is specific to each DB + # driver + + def test_capabilities(self): + expected_capabilities = { + 'meters': {'pagination': False, + 'query': {'simple': True, + 'metadata': True, + 'complex': False}}, + 'resources': {'pagination': False, + 'query': {'simple': True, + 'metadata': True, + 'complex': False}}, + 'samples': {'pagination': False, + 'groupby': False, + 'query': {'simple': True, + 'metadata': True, + 'complex': True}}, + 'statistics': {'pagination': False, + 'groupby': True, + 'query': {'simple': True, + 'metadata': True, + 'complex': False}, + 'aggregation': {'standard': True, + 'selectable': { + 'max': False, + 'min': False, + 'sum': False, + 'avg': False, + 'count': False, + 'stddev': False, + 'cardinality': False}} + }, + 'alarms': {'query': {'simple': True, + 'complex': True}, + 'history': {'query': {'simple': True, + 'complex': False}}}, + 'events': {'query': {'simple': False}} + } + + actual_capabilities = self.conn.get_capabilities() + self.assertEqual(expected_capabilities, actual_capabilities) diff --git a/ceilometer/tests/storage/test_impl_hbase.py b/ceilometer/tests/storage/test_impl_hbase.py index d4e5cf188..a89189fd5 100644 --- a/ceilometer/tests/storage/test_impl_hbase.py +++ b/ceilometer/tests/storage/test_impl_hbase.py @@ -56,3 +56,48 @@ class ConnectionTest(HBaseEngineTestBase): side_effect=get_connection): conn = hbase.Connection(self.CONF) self.assertIsInstance(conn.conn, TestConn) + + +class CapabilitiesTest(HBaseEngineTestBase): + # Check the returned capabilities list, which is specific to each DB + # driver + + def test_capabilities(self): + expected_capabilities = { + 'meters': {'pagination': False, + 'query': {'simple': True, + 'metadata': True, + 'complex': False}}, + 'resources': {'pagination': False, + 'query': {'simple': True, + 'metadata': True, + 'complex': False}}, + 'samples': {'pagination': False, + 'groupby': False, + 'query': {'simple': True, + 'metadata': True, + 'complex': False}}, + 'statistics': {'pagination': False, + 'groupby': False, + 'query': {'simple': True, + 'metadata': True, + 'complex': False}, + 'aggregation': {'standard': True, + 'selectable': { + 'max': False, + 'min': False, + 'sum': False, + 'avg': False, + 'count': False, + 'stddev': False, + 'cardinality': False}} + }, + 'alarms': {'query': {'simple': False, + 'complex': False}, + 'history': {'query': {'simple': False, + 'complex': False}}}, + 'events': {'query': {'simple': False}} + } + + actual_capabilities = self.conn.get_capabilities() + self.assertEqual(expected_capabilities, actual_capabilities) diff --git a/ceilometer/tests/storage/test_impl_mongodb.py b/ceilometer/tests/storage/test_impl_mongodb.py index 02709e5bc..a48bb4ee4 100644 --- a/ceilometer/tests/storage/test_impl_mongodb.py +++ b/ceilometer/tests/storage/test_impl_mongodb.py @@ -295,3 +295,48 @@ class AlarmTestPagination(test_storage_scenarios.AlarmTestBase, 'counter-name-foo') except base.MultipleResultsFound: self.assertTrue(True) + + +class CapabilitiesTest(MongoDBEngineTestBase): + # Check the returned capabilities list, which is specific to each DB + # driver + + def test_capabilities(self): + expected_capabilities = { + 'meters': {'pagination': False, + 'query': {'simple': True, + 'metadata': True, + 'complex': False}}, + 'resources': {'pagination': False, + 'query': {'simple': True, + 'metadata': True, + 'complex': False}}, + 'samples': {'pagination': False, + 'groupby': False, + 'query': {'simple': True, + 'metadata': True, + 'complex': True}}, + 'statistics': {'pagination': False, + 'groupby': True, + 'query': {'simple': True, + 'metadata': True, + 'complex': False}, + 'aggregation': {'standard': True, + 'selectable': { + 'max': True, + 'min': True, + 'sum': True, + 'avg': True, + 'count': True, + 'stddev': True, + 'cardinality': True}} + }, + 'alarms': {'query': {'simple': True, + 'complex': True}, + 'history': {'query': {'simple': True, + 'complex': True}}}, + 'events': {'query': {'simple': False}} + } + + actual_capabilities = self.conn.get_capabilities() + self.assertEqual(expected_capabilities, actual_capabilities) diff --git a/ceilometer/tests/storage/test_impl_sqlalchemy.py b/ceilometer/tests/storage/test_impl_sqlalchemy.py index 7a9fd6d55..77b8de3cf 100644 --- a/ceilometer/tests/storage/test_impl_sqlalchemy.py +++ b/ceilometer/tests/storage/test_impl_sqlalchemy.py @@ -222,3 +222,48 @@ class RelationshipTest(scenarios.DBTestBase): session.query(sql_models.User.id) .group_by(sql_models.User.id) )).count(), 0) + + +class CapabilitiesTest(EventTestBase): + # Check the returned capabilities list, which is specific to each DB + # driver + + def test_capabilities(self): + expected_capabilities = { + 'meters': {'pagination': False, + 'query': {'simple': True, + 'metadata': True, + 'complex': False}}, + 'resources': {'pagination': False, + 'query': {'simple': True, + 'metadata': True, + 'complex': False}}, + 'samples': {'pagination': True, + 'groupby': True, + 'query': {'simple': True, + 'metadata': True, + 'complex': True}}, + 'statistics': {'pagination': False, + 'groupby': True, + 'query': {'simple': True, + 'metadata': True, + 'complex': False}, + 'aggregation': {'standard': True, + 'selectable': { + 'max': True, + 'min': True, + 'sum': True, + 'avg': True, + 'count': True, + 'stddev': True, + 'cardinality': True}} + }, + 'alarms': {'query': {'simple': True, + 'complex': True}, + 'history': {'query': {'simple': True, + 'complex': True}}}, + 'events': {'query': {'simple': True}} + } + + actual_capabilities = self.conn.get_capabilities() + self.assertEqual(expected_capabilities, actual_capabilities) diff --git a/ceilometer/utils.py b/ceilometer/utils.py index bf869223b..e7301b8ac 100644 --- a/ceilometer/utils.py +++ b/ceilometer/utils.py @@ -19,6 +19,7 @@ """Utilities and helper functions.""" import calendar +import copy import datetime import decimal @@ -134,14 +135,15 @@ def lowercase_values(mapping): mapping[key] = value.lower() -def update_nested(d, u): +def update_nested(original_dict, updates): """Updates the leaf nodes in a nest dict, without replacing entire sub-dicts. """ - for k, v in u.iteritems(): - if isinstance(v, dict): - r = update_nested(d.get(k, {}), v) - d[k] = r + dict_to_update = copy.deepcopy(original_dict) + for key, value in updates.iteritems(): + if isinstance(value, dict): + sub_dict = update_nested(dict_to_update.get(key, {}), value) + dict_to_update[key] = sub_dict else: - d[k] = u[k] - return d + dict_to_update[key] = updates[key] + return dict_to_update