diff --git a/ceilometer/storage/impl_mongodb.py b/ceilometer/storage/impl_mongodb.py index b8dd11883..be7a58d3f 100644 --- a/ceilometer/storage/impl_mongodb.py +++ b/ceilometer/storage/impl_mongodb.py @@ -21,7 +21,6 @@ """ import copy -import datetime import operator import os import re @@ -155,11 +154,32 @@ def make_query_from_filter(sample_filter, require_meter=True): return q +class ConnectionPool(object): + + def __init__(self): + self._pool = {} + + def connect(self, opts): + # opts is a dict, dict are unhashable, convert to tuple + connection_pool_key = tuple(sorted(opts.items())) + + if connection_pool_key not in self._pool: + LOG.info('connecting to MongoDB replicaset "%s" on %s', + opts['replica_set'], + opts['netloc']) + self._pool[connection_pool_key] = pymongo.Connection( + opts['netloc'], + replicaSet=opts['replica_set'], + safe=True) + + return self._pool.get(connection_pool_key) + + class Connection(base.Connection): """MongoDB connection. """ - _mim_instance = None + CONNECTION_POOL = ConnectionPool() MAP_STATS = bson.code.Code(""" function () { @@ -196,13 +216,20 @@ class Connection(base.Connection): REDUCE_STATS = bson.code.Code(""" function (key, values) { - var res = values[0]; + var res = { min: values[0].min, + max: values[0].max, + count: values[0].count, + sum: values[0].sum, + period_start: values[0].period_start, + period_end: values[0].period_end, + duration_start: values[0].duration_start, + duration_end: values[0].duration_end }; for ( var i=1; i res.max ) res.max = values[i].max; - res.count += values[i].count; + res.count = NumberInt(res.count + values[i].count); res.sum += values[i].sum; if ( values[i].duration_start < res.duration_start ) res.duration_start = values[i].duration_start; @@ -224,33 +251,23 @@ class Connection(base.Connection): def __init__(self, conf): opts = self._parse_connection_url(conf.database.connection) - LOG.info('connecting to MongoDB replicaset "%s" on %s', - conf.storage_mongodb.replica_set_name, - opts['netloc']) if opts['netloc'] == '__test__': url = os.environ.get('CEILOMETER_TEST_MONGODB_URL') - if url: - opts = self._parse_connection_url(url) - self.conn = pymongo.Connection(opts['netloc'], safe=True) - else: - # MIM will die if we have too many connections, so use a - # Singleton - if Connection._mim_instance is None: - try: - from ming import mim - except ImportError: - import testtools - raise testtools.testcase.TestSkipped('requires mim') - LOG.debug('Creating a new MIM Connection object') - Connection._mim_instance = mim.Connection() - self.conn = Connection._mim_instance - LOG.debug('Using MIM for test connection') - else: - self.conn = pymongo.Connection( - opts['netloc'], - replicaSet=conf.storage_mongodb.replica_set_name, - safe=True) + if not url: + raise RuntimeError( + "No MongoDB test URL set," + "export CEILOMETER_TEST_MONGODB_URL environment variable") + opts = self._parse_connection_url(url) + + # FIXME(jd) This should be a parameter in the database URL, not global + opts['replica_set'] = conf.storage_mongodb.replica_set_name + + # NOTE(jd) Use our own connection pooling on top of the Pymongo one. + # We need that otherwise we overflow the MongoDB instance with new + # connection since we instanciate a Pymongo client each time someone + # requires a new storage connection. + self.conn = self.CONNECTION_POOL.connect(opts) self.db = getattr(self.conn, opts['dbname']) if 'username' in opts: @@ -281,13 +298,7 @@ class Connection(base.Connection): pass def clear(self): - if self._mim_instance is not None: - # Don't want to use drop_database() because - # may end up running out of spidermonkey instances. - # http://davisp.lighthouseapp.com/projects/26898/tickets/22 - self.db.clear() - else: - self.conn.drop_database(self.db) + self.conn.drop_database(self.db) @staticmethod def _parse_connection_url(url): @@ -526,34 +537,6 @@ class Connection(base.Connection): for r in results['results']), key=operator.attrgetter('period_start')) - def _fix_interval_min_max(self, a_min, a_max): - if hasattr(a_min, 'valueOf') and a_min.valueOf is not None: - # NOTE (dhellmann): HACK ALERT - # - # The real MongoDB server can handle Date objects and - # the driver converts them to datetime instances - # correctly but the in-memory implementation in MIM - # (used by the tests) returns a spidermonkey.Object - # representing the "value" dictionary and there - # doesn't seem to be a way to recursively introspect - # that object safely to convert the min and max values - # back to datetime objects. In this method, we know - # what type the min and max values are expected to be, - # so it is safe to do the conversion - # here. JavaScript's time representation uses - # different units than Python's, so we divide to - # convert to the right units and then create the - # datetime instances to return. - # - # The issue with MIM is documented at - # https://sourceforge.net/p/merciless/bugs/3/ - # - a_min = datetime.datetime.fromtimestamp( - a_min.valueOf() // 1000) - a_max = datetime.datetime.fromtimestamp( - a_max.valueOf() // 1000) - return (a_min, a_max) - def get_alarms(self, name=None, user=None, project=None, enabled=True, alarm_id=None): """Yields a lists of alarms that match filters @@ -612,22 +595,3 @@ class Connection(base.Connection): :param event_filter: EventFilter instance """ raise NotImplementedError('Events not implemented.') - - -def require_map_reduce(conn): - """Raises SkipTest if the connection is using mim. - """ - # NOTE(dhellmann): mim requires spidermonkey to implement the - # map-reduce functions, so if we can't import it then just - # skip these tests unless we aren't using mim. - try: - import spidermonkey # noqa - except BaseException: - try: - from ming import mim - if hasattr(conn, "conn") and isinstance(conn.conn, mim.Connection): - import testtools - raise testtools.testcase.TestSkipped('requires spidermonkey') - except ImportError: - import testtools - raise testtools.testcase.TestSkipped('requires mim') diff --git a/run-tests.sh b/run-tests.sh new file mode 100755 index 000000000..12d9e6613 --- /dev/null +++ b/run-tests.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -e + +# Nova notifier tests +bash tools/init_testr_if_needed.sh +python setup.py testr --slowest --testr-args="--concurrency=1 --here=nova_tests $*" + +# Main unit tests +MONGO_DATA=`mktemp -d` +trap "rm -rf ${MONGO_DATA}" EXIT +mongod --maxConns 32 --smallfiles --quiet --noauth --port 29000 --dbpath "${MONGO_DATA}" --bind_ip localhost & +MONGO_PID=$! +trap "kill -9 ${MONGO_PID} || true" EXIT +export CEILOMETER_TEST_MONGODB_URL="mongodb://localhost:29000/ceilometer" +python setup.py testr --slowest --testr-args="--concurrency=1 $*" diff --git a/test-requirements.txt b/test-requirements.txt index dff1f1951..b3c45203b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,9 +5,6 @@ mock mox fixtures>=0.3.12 Babel>=0.9.6 -# NOTE(dhellmann): Ming is necessary to provide the Mongo-in-memory -# implementation of MongoDB. -Ming>=0.3.4 http://tarballs.openstack.org/nova/nova-master.tar.gz#egg=nova # We should use swift>1.7.5, but it's not yet available swift @@ -18,7 +15,6 @@ sphinx sphinxcontrib-pecanwsme>=0.2 docutils==0.9.1 # for bug 1091333, remove after sphinx >1.1.3 is released. oslo.sphinx -python-spidermonkey python-subunit testrepository>=0.0.13 testtools>=0.9.29 diff --git a/tests/api/v1/max_project_volume.py b/tests/api/v1/max_project_volume.py index d576f7359..20c9672ff 100644 --- a/tests/api/v1/max_project_volume.py +++ b/tests/api/v1/max_project_volume.py @@ -27,14 +27,12 @@ from ceilometer.publisher import rpc from ceilometer import counter from ceilometer.tests import api as tests_api -from ceilometer.storage.impl_mongodb import require_map_reduce class TestMaxProjectVolume(tests_api.TestBase): def setUp(self): super(TestMaxProjectVolume, self).setUp() - require_map_reduce(self.conn) self.counters = [] for i in range(3): diff --git a/tests/api/v1/max_resource_volume.py b/tests/api/v1/max_resource_volume.py index d134bc4a5..478fb74ce 100644 --- a/tests/api/v1/max_resource_volume.py +++ b/tests/api/v1/max_resource_volume.py @@ -26,14 +26,12 @@ from ceilometer.publisher import rpc from ceilometer import counter from ceilometer.tests import api as tests_api -from ceilometer.storage.impl_mongodb import require_map_reduce class TestMaxResourceVolume(tests_api.TestBase): def setUp(self): super(TestMaxResourceVolume, self).setUp() - require_map_reduce(self.conn) self.counters = [] for i in range(3): diff --git a/tests/api/v1/sum_project_volume.py b/tests/api/v1/sum_project_volume.py index 60b9f849e..e8d57b189 100644 --- a/tests/api/v1/sum_project_volume.py +++ b/tests/api/v1/sum_project_volume.py @@ -27,14 +27,12 @@ from ceilometer.publisher import rpc from ceilometer import counter from ceilometer.tests import api as tests_api -from ceilometer.storage.impl_mongodb import require_map_reduce class TestSumProjectVolume(tests_api.TestBase): def setUp(self): super(TestSumProjectVolume, self).setUp() - require_map_reduce(self.conn) self.counters = [] for i in range(3): diff --git a/tests/api/v1/sum_resource_volume.py b/tests/api/v1/sum_resource_volume.py index 3e160fc02..0c3fe03ed 100644 --- a/tests/api/v1/sum_resource_volume.py +++ b/tests/api/v1/sum_resource_volume.py @@ -27,14 +27,12 @@ from ceilometer.publisher import rpc from ceilometer import counter from ceilometer.tests import api as tests_api -from ceilometer.storage.impl_mongodb import require_map_reduce class TestSumResourceVolume(tests_api.TestBase): def setUp(self): super(TestSumResourceVolume, self).setUp() - require_map_reduce(self.conn) self.counters = [] for i in range(3): diff --git a/tests/api/v2/statistics.py b/tests/api/v2/statistics.py index 137a9ccde..6d5f4dc45 100644 --- a/tests/api/v2/statistics.py +++ b/tests/api/v2/statistics.py @@ -23,8 +23,6 @@ from oslo.config import cfg from . import base from ceilometer import counter -from ceilometer.storage.impl_mongodb import Connection as mongo_conn -from ceilometer.storage.impl_mongodb import require_map_reduce from ceilometer.publisher import rpc @@ -34,9 +32,6 @@ class TestMaxProjectVolume(base.FunctionalTest): def setUp(self): super(TestMaxProjectVolume, self).setUp() - # TODO(gordc): remove when we drop mim - if isinstance(self.conn, mongo_conn): - require_map_reduce(self.conn) self.counters = [] for i in range(3): @@ -137,9 +132,6 @@ class TestMaxResourceVolume(base.FunctionalTest): def setUp(self): super(TestMaxResourceVolume, self).setUp() - # TODO(gordc): remove when we drop mim - if isinstance(self.conn, mongo_conn): - require_map_reduce(self.conn) self.counters = [] for i in range(3): @@ -256,9 +248,6 @@ class TestSumProjectVolume(base.FunctionalTest): def setUp(self): super(TestSumProjectVolume, self).setUp() - # TODO(gordc): remove when we drop mim - if isinstance(self.conn, mongo_conn): - require_map_reduce(self.conn) self.counters = [] for i in range(3): @@ -361,9 +350,6 @@ class TestSumResourceVolume(base.FunctionalTest): def setUp(self): super(TestSumResourceVolume, self).setUp() - # TODO(gordc): remove when we drop mim - if isinstance(self.conn, mongo_conn): - require_map_reduce(self.conn) self.counters = [] for i in range(3): diff --git a/tests/storage/test_impl_mongodb.py b/tests/storage/test_impl_mongodb.py index 8e891c3ff..5dff9af69 100644 --- a/tests/storage/test_impl_mongodb.py +++ b/tests/storage/test_impl_mongodb.py @@ -18,56 +18,31 @@ """Tests for ceilometer/storage/impl_mongodb.py .. note:: - - (dhellmann) These tests have some dependencies which cannot be - installed in the CI environment right now. - - Ming is necessary to provide the Mongo-in-memory implementation for - of MongoDB. The original source for Ming is at - http://sourceforge.net/project/merciless but there does not seem to - be a way to point to a "zipball" of the latest HEAD there, and we - need features present only in that version. I forked the project to - github to make it easier to install, and put the URL into the - test-requires file. Then I ended up making some changes to it so it - would be compatible with PyMongo's API. - - https://github.com/dreamhost/Ming/zipball/master#egg=Ming - - In order to run the tests that use map-reduce with MIM, some - additional system-level packages are required:: - - apt-get install nspr-config - apt-get install libnspr4-dev - apt-get install pkg-config - pip install python-spidermonkey - - To run the tests *without* mim, set the environment variable - CEILOMETER_TEST_MONGODB_URL to a MongoDB URL before running tox. + In order to run the tests against another MongoDB server set the + environment variable CEILOMETER_TEST_MONGODB_URL to point to a MongoDB + server before running the tests. """ import copy import datetime +from oslo.config import cfg from tests.storage import base from ceilometer.publisher import rpc from ceilometer import counter -from ceilometer.storage.impl_mongodb import require_map_reduce +from ceilometer.storage import impl_mongodb class MongoDBEngineTestBase(base.DBTestBase): database_connection = 'mongodb://__test__' -class IndexTest(MongoDBEngineTestBase): - - def test_indexes_exist(self): - # ensure_index returns none if index already exists - assert not self.conn.db.resource.ensure_index('foo', - name='resource_idx') - assert not self.conn.db.meter.ensure_index('foo', - name='meter_idx') +class MongoDBConnection(MongoDBEngineTestBase): + def test_connection_pooling(self): + self.assertEqual(self.conn.conn, + impl_mongodb.Connection(cfg.CONF).conn) class UserTest(base.UserTest, MongoDBEngineTestBase): @@ -91,10 +66,7 @@ class RawSampleTest(base.RawSampleTest, MongoDBEngineTestBase): class StatisticsTest(base.StatisticsTest, MongoDBEngineTestBase): - - def setUp(self): - super(StatisticsTest, self).setUp() - require_map_reduce(self.conn) + pass class AlarmTest(base.AlarmTest, MongoDBEngineTestBase): diff --git a/tox.ini b/tox.ini index 170e67ae0..141ccd028 100644 --- a/tox.ini +++ b/tox.ini @@ -7,10 +7,9 @@ deps = -r{toxinidir}/requirements.txt setenv = VIRTUAL_ENV={envdir} EVENTLET_NO_GREENDNS=yes commands = - python setup.py testr --slowest --testr-args='--concurrency=1 {posargs}' - bash tools/init_testr_if_needed.sh - python setup.py testr --slowest --testr-args='--concurrency=1 --here=nova_tests {posargs}' + bash -x {toxinidir}/run-tests.sh {posargs} {toxinidir}/tools/conf/check_uptodate.sh + sitepackages = False downloadcache = {toxworkdir}/_download