From 5983a99da96a64fa37cf5965bef81e3b24ddce2a Mon Sep 17 00:00:00 2001 From: Alexei Kornienko Date: Thu, 25 Jul 2013 13:51:48 +0300 Subject: [PATCH] Added separate MongoDB database for each test This allows to run all the tests concurrently Change-Id: If29c09e6f8e5e0e7c3ae98bd9a71842dcfb43011 --- ceilometer/storage/impl_mongodb.py | 10 ++-------- ceilometer/tests/db.py | 24 ++++++++++++++++++++++-- run-tests.sh | 6 +++--- tests/api/v2/test_app.py | 3 ++- tests/storage/test_impl_mongodb.py | 6 +++--- 5 files changed, 32 insertions(+), 17 deletions(-) diff --git a/ceilometer/storage/impl_mongodb.py b/ceilometer/storage/impl_mongodb.py index d973879b3..1da5c5eb3 100644 --- a/ceilometer/storage/impl_mongodb.py +++ b/ceilometer/storage/impl_mongodb.py @@ -24,7 +24,6 @@ import calendar import copy import datetime import operator -import os import uuid import weakref @@ -247,13 +246,6 @@ class Connection(base.Connection): def __init__(self, conf): url = conf.database.connection - if url == 'mongodb://__test__': - url = os.environ.get('CEILOMETER_TEST_MONGODB_URL') - if not url: - raise RuntimeError( - "No MongoDB test URL set," - "export CEILOMETER_TEST_MONGODB_URL environment variable") - # NOTE(jd) Use our own connection pooling on top of the Pymongo one. # We need that otherwise we overflow the MongoDB instance with new # connection since we instanciate a Pymongo client each time someone @@ -326,6 +318,8 @@ class Connection(base.Connection): def clear(self): self.conn.drop_database(self.db) + # Connection will be reopened automatically if needed + self.conn.close() def record_metering_data(self, data): """Write the data to the backend storage system. diff --git a/ceilometer/tests/db.py b/ceilometer/tests/db.py index 1d8ab6761..4ed01302f 100644 --- a/ceilometer/tests/db.py +++ b/ceilometer/tests/db.py @@ -19,6 +19,8 @@ # under the License. """Base classes for API tests.""" +import os +import uuid from oslo.config import cfg @@ -29,19 +31,37 @@ from ceilometer.tests import base as test_base class TestBase(test_base.TestCase): def setUp(self): super(TestBase, self).setUp() - cfg.CONF.set_override('connection', self.database_connection, + cfg.CONF.set_override('connection', str(self.database_connection), group='database') self.conn = storage.get_connection(cfg.CONF) self.conn.upgrade() self.conn.clear() self.conn.upgrade() + def tearDown(self): + self.conn.clear() + self.conn = None + super(TestBase, self).tearDown() + + +class MongoDBFakeConnectionUrl(object): + + def __init__(self): + self.url = os.environ.get('CEILOMETER_TEST_MONGODB_URL') + if not self.url: + raise RuntimeError( + "No MongoDB test URL set," + "export CEILOMETER_TEST_MONGODB_URL environment variable") + + def __str__(self): + return '%(url)s_%(db)s' % dict(url=self.url, db=uuid.uuid4().hex) + class MixinTestsWithBackendScenarios(object): __metaclass__ = test_base.SkipNotImplementedMeta scenarios = [ ('sqlalchemy', dict(database_connection='sqlite://')), - ('mongodb', dict(database_connection='mongodb://__test__')), + ('mongodb', dict(database_connection=MongoDBFakeConnectionUrl())), ('hbase', dict(database_connection='hbase://__test__')), ] diff --git a/run-tests.sh b/run-tests.sh index 62fd94bd7..69dd753a8 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -25,7 +25,7 @@ fi MONGO_DATA=`mktemp -d` trap "clean_exit" EXIT mkfifo ${MONGO_DATA}/out -mongod --maxConns 32 --noprealloc --smallfiles --quiet --noauth --port 29000 --dbpath "${MONGO_DATA}" --bind_ip localhost &>${MONGO_DATA}/out & +mongod --maxConns 32 --nojournal --noprealloc --smallfiles --quiet --noauth --port 29000 --dbpath "${MONGO_DATA}" --bind_ip localhost &>${MONGO_DATA}/out & MONGO_PID=$! # Wait for Mongo to start listening to connections while read line @@ -34,6 +34,6 @@ do done < ${MONGO_DATA}/out # Read the fifo for ever otherwise mongod would block # + that gives us the log on screen -cat ${MONGO_DATA}/out & +cat ${MONGO_DATA}/out > /dev/null & export CEILOMETER_TEST_MONGODB_URL="mongodb://localhost:29000/ceilometer" -python setup.py testr --slowest --testr-args="--concurrency=1 $*" $COVERAGE_ARG +python setup.py testr --slowest --testr-args="$*" $COVERAGE_ARG diff --git a/tests/api/v2/test_app.py b/tests/api/v2/test_app.py index 3b0b41d24..738022a82 100644 --- a/tests/api/v2/test_app.py +++ b/tests/api/v2/test_app.py @@ -25,6 +25,7 @@ from ceilometer.api import app from ceilometer.api import acl from ceilometer import service from ceilometer.tests import base +from ceilometer.tests import db as tests_db from .base import FunctionalTest @@ -62,7 +63,7 @@ class TestApp(base.TestCase): class TestApiMiddleware(FunctionalTest): # This doesn't really matter - database_connection = 'mongodb://__test__' + database_connection = tests_db.MongoDBFakeConnectionUrl() def test_json_parsable_error_middleware_404(self): response = self.get_json('/invalid_path', diff --git a/tests/storage/test_impl_mongodb.py b/tests/storage/test_impl_mongodb.py index 5449272e5..e8390ea44 100644 --- a/tests/storage/test_impl_mongodb.py +++ b/tests/storage/test_impl_mongodb.py @@ -36,10 +36,11 @@ from ceilometer.publisher import rpc from ceilometer import counter from ceilometer.storage import impl_mongodb from ceilometer.storage import models +from ceilometer.tests import db as tests_db class MongoDBEngineTestBase(base.DBTestBase): - database_connection = 'mongodb://__test__' + database_connection = tests_db.MongoDBFakeConnectionUrl() class MongoDBConnection(MongoDBEngineTestBase): @@ -48,10 +49,9 @@ class MongoDBConnection(MongoDBEngineTestBase): impl_mongodb.Connection(cfg.CONF).conn) def test_replica_set(self): - # FIXME(Alexei_987) should not hardcode URL here cfg.CONF.set_override( 'connection', - 'mongodb://localhost:29000/ceilometer?replicaSet=foobar', + str(tests_db.MongoDBFakeConnectionUrl()) + '?replicaSet=foobar', group='database') conn = impl_mongodb.Connection(cfg.CONF) self.assertTrue(conn.conn)