openstack/aodh
Code Issues Proposed changes

Remove impl_test

Browse Source 
Change-Id: If939bbaebf35a326076b6263f70248ff13975b45
Signed-off-by: Julien Danjou <julien@danjou.info>
This commit is contained in:
Julien Danjou 2013-03-22 18:00:12 +01:00
parent 823f2005d8
commit a284960c37
16 changed files with 88 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
ceilometer/storage/impl_mongodb.py
View File

@ -1,8 +1,10 @@
# -*- encoding: utf-8 -*-
#
# Copyright © 2012 New Dream Network, LLC (DreamHost)
# Copyright © 2013 eNovance
#
# Author: Doug Hellmann <doug.hellmann@dreamhost.com>
# Julien Danjou <julien@danjou.info>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -20,7 +22,9 @@
import copy
import datetime
import nose
import operator
import os
import re
import urlparse
@ -126,6 +130,8 @@ class Connection(base.Connection):
"""MongoDB connection.
"""
_mim_instance = None
# JavaScript function for doing map-reduce to get a counter volume
# total.
MAP_COUNTER_VOLUME = bson.code.Code("""
@ -240,7 +246,36 @@ class Connection(base.Connection):
def __init__(self, conf):
opts = self._parse_connection_url(conf.database_connection)
LOG.info('connecting to MongoDB on %s:%s', opts['host'], opts['port'])
self.conn = self._get_connection(opts)
if opts['host'] == '__test__':
live_tests = bool(int(os.environ.get('CEILOMETER_TEST_LIVE', 0)))
if live_tests:
url = os.environ.get('CEILOMETER_TEST_MONGO_URL')
if not url:
raise RuntimeError("CEILOMETER_TEST_LIVE is on, but "
"CEILOMETER_TEST_MONGO_URL "
"is not defined")
opts = self._parse_connection_url(url)
self.conn = pymongo.Connection(opts['host'],
opts['port'],
safe=True)
else:
# MIM will die if we have too many connections, so use a
# Singleton
if Connection._mim_instance is None:
try:
from ming import mim
except ImportError:
raise nose.SkipTest("Ming not found")
LOG.debug('Creating a new MIM Connection object')
Connection._mim_instance = mim.Connection()
self.conn = Connection._mim_instance
LOG.debug('Using MIM for test connection')
else:
self.conn = pymongo.Connection(opts['host'],
opts['port'],
safe=True)
self.db = getattr(self.conn, opts['dbname'])
if 'username' in opts:
self.db.authenticate(opts['username'], opts['password'])
@ -269,17 +304,13 @@ class Connection(base.Connection):
pass
def clear(self):
self.conn.drop_database(self.db)
def _get_connection(self, opts):
"""Return a connection to the database.
.. note::
The tests use a subclass to override this and return an
in-memory connection.
"""
return pymongo.Connection(opts['host'], opts['port'], safe=True)
if self._mim_instance is not None:
# Don't want to use drop_database() because
# may end up running out of spidermonkey instances.
# http://davisp.lighthouseapp.com/projects/26898/tickets/22
self.db.clear()
else:
self.conn.drop_database(self.db)
def _parse_connection_url(self, url):
opts = {}
@ -592,3 +623,16 @@ class Connection(base.Connection):
answer = results['results'][0]['value']
return self._fix_interval_min_max(answer['min'], answer['max'])
return (None, None)
def require_map_reduce(conn):
"""Raises SkipTest if the connection is using mim.
"""
# NOTE(dhellmann): mim requires spidermonkey to implement the
# map-reduce functions, so if we can't import it then just
# skip these tests unless we aren't using mim.
try:
import spidermonkey
except BaseException:
if isinstance(conn.conn, mim.Connection):
raise nose.SkipTest('requires spidermonkey')

129
ceilometer/storage/impl_test.py
View File

@ -1,129 +0,0 @@
# -*- encoding: utf-8 -*-
#
# Copyright © 2012 New Dream Network, LLC (DreamHost)
#
# Author: Doug Hellmann <doug.hellmann@dreamhost.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""In-memory storage driver for use with tests.
This driver is based on MIM, an in-memory version of MongoDB.
"""
import os
import nose
try:
from ming import mim
except ImportError:
mim = None
from ceilometer.openstack.common import log as logging
from ceilometer.storage import base
from ceilometer.storage import impl_mongodb
LOG = logging.getLogger(__name__)
class TestDBStorage(base.StorageEngine):
"""Put the data into an in-memory database for testing
This driver is based on MIM, an in-memory version of MongoDB.
Collections::
- user
- { _id: user id
source: [ array of source ids reporting for the user ]
}
- project
- { _id: project id
source: [ array of source ids reporting for the project ]
}
- meter
- the raw incoming data
- resource
- the metadata for resources
- { _id: uuid of resource,
metadata: metadata dictionaries
timestamp: datetime of last update
user_id: uuid
project_id: uuid
meter: [ array of {counter_name: string, counter_type: string,
counter_unit: string} ]
}
"""
OPTIONS = []
def register_opts(self, conf):
"""Register any configuration options used by this engine.
"""
conf.register_opts(self.OPTIONS)
def get_connection(self, conf):
"""Return a Connection instance based on the configuration settings.
"""
return TestConnection(conf)
class TestConnection(impl_mongodb.Connection):
_mim_instance = None
FORCE_MONGO = bool(int(os.environ.get('CEILOMETER_TEST_LIVE', 0)))
def clear(self):
if TestConnection._mim_instance is not None:
# Don't want to use drop_database() because
# may end up running out of spidermonkey instances.
# http://davisp.lighthouseapp.com/projects/26898/tickets/22
self.db.clear()
else:
super(TestConnection, self).clear()
def _get_connection(self, conf):
# Use a real MongoDB server if we can connect, but fall back
# to a Mongo-in-memory connection if we cannot.
if self.FORCE_MONGO:
try:
return super(TestConnection, self)._get_connection(conf)
except:
LOG.debug('Unable to connect to mongodb')
raise
else:
LOG.debug('Using MIM for test connection')
# MIM will die if we have too many connections, so use a
# Singleton
if TestConnection._mim_instance is None:
if mim:
LOG.debug('Creating a new MIM Connection object')
TestConnection._mim_instance = mim.Connection()
else:
raise nose.SkipTest("Ming not found")
return TestConnection._mim_instance
def require_map_reduce(conn):
"""Raises SkipTest if the connection is using mim.
"""
# NOTE(dhellmann): mim requires spidermonkey to implement the
# map-reduce functions, so if we can't import it then just
# skip these tests unless we aren't using mim.
try:
import spidermonkey
except BaseException:
if isinstance(conn.conn, mim.Connection):
raise skip.SkipTest('requires spidermonkey')

2
ceilometer/tests/api.py
View File

@ -30,8 +30,6 @@ import pecan.testing
from ceilometer.api import acl
from ceilometer.api.v1 import app as v1_app
from ceilometer.api.v1 import blueprint as v1_blueprint
from ceilometer import storage
from ceilometer.tests import base
from ceilometer.tests import db as db_test_base

6
ceilometer/tests/db.py
View File

@ -33,8 +33,10 @@ class BaseException(Exception):
class TestBase(test_base.TestCase):
# Default tests use test:// (MIM)
database_connection = 'test://'
# Default tests use mongodb://__test__ (MIM)
# TODO(jd) remove it, so we're sure we run test on the backend we want,
# not this default one by mistake
database_connection = 'mongodb://__test__'
def setUp(self):
super(TestBase, self).setUp()

1
setup.py
View File

@ -130,7 +130,6 @@ setuptools.setup(
mysql = ceilometer.storage.impl_sqlalchemy:SQLAlchemyStorage
postgresql = ceilometer.storage.impl_sqlalchemy:SQLAlchemyStorage
sqlite = ceilometer.storage.impl_sqlalchemy:SQLAlchemyStorage
test = ceilometer.storage.impl_test:TestDBStorage
hbase = ceilometer.storage.impl_hbase:HBaseStorage
[ceilometer.compute.virt]

2
tests/api/v1/max_project_volume.py
View File

@ -27,7 +27,7 @@ from ceilometer.collector import meter
from ceilometer import counter
from ceilometer.tests import api as tests_api
from ceilometer.storage.impl_test import require_map_reduce
from ceilometer.storage.impl_mongodb import require_map_reduce
class TestMaxProjectVolume(tests_api.TestBase):

2
tests/api/v1/max_resource_volume.py
View File

@ -26,7 +26,7 @@ from ceilometer.collector import meter
from ceilometer import counter
from ceilometer.tests import api as tests_api
from ceilometer.storage.impl_test import require_map_reduce
from ceilometer.storage.impl_mongodb import require_map_reduce
class TestMaxResourceVolume(tests_api.TestBase):

32
tests/api/v1/test_impl_mongodb.py
View File

@ -29,64 +29,64 @@ from . import max_resource_volume
class TestListEvents(list_events.TestListEvents):
database_connection = 'test://'
database_connection = 'mongodb://__test__'
class TestListEventsMetaQuery(list_events.TestListEventsMetaquery):
database_connection = 'test://'
database_connection = 'mongodb://__test__'
class TestListEmptyMeters(list_meters.TestListEmptyMeters):
database_connection = 'test://'
database_connection = 'mongodb://__test__'
class TestListMeters(list_meters.TestListMeters):
database_connection = 'test://'
database_connection = 'mongodb://__test__'
class TestListMetersMetaquery(list_meters.TestListMetersMetaquery):
database_connection = 'test://'
database_connection = 'mongodb://__test__'
class TestListEmptyUsers(list_users.TestListEmptyUsers):
database_connection = 'test://'
database_connection = 'mongodb://__test__'
class TestListUsers(list_users.TestListUsers):
database_connection = 'test://'
database_connection = 'mongodb://__test__'
class TestListEmptyProjects(list_projects.TestListEmptyProjects):
database_connection = 'test://'
database_connection = 'mongodb://__test__'
class TestListProjects(list_projects.TestListProjects):
database_connection = 'test://'
database_connection = 'mongodb://__test__'
class TestComputeDurationByResource(cdbr.TestComputeDurationByResource):
database_connection = 'test://'
database_connection = 'mongodb://__test__'
class TestListEmptyResources(list_resources.TestListEmptyResources):
database_connection = 'test://'
database_connection = 'mongodb://__test__'
class TestListResources(list_resources.TestListResources):
database_connection = 'test://'
database_connection = 'mongodb://__test__'
class TestListResourcesMetaquery(list_resources.TestListResourcesMetaquery):
database_connection = 'test://'
database_connection = 'mongodb://__test__'
class TestListSource(list_sources.TestListSource):
database_connection = 'test://'
database_connection = 'mongodb://__test__'
class TestMaxProjectVolume(max_project_volume.TestMaxProjectVolume):
database_connection = 'test://'
database_connection = 'mongodb://__test__'
class TestMaxProjectVolume(max_resource_volume.TestMaxResourceVolume):
database_connection = 'test://'
database_connection = 'mongodb://__test__'

2
tests/api/v1/test_sum_project_volume.py
View File

@ -27,7 +27,7 @@ from ceilometer.collector import meter
from ceilometer import counter
from ceilometer.tests import api as tests_api
from ceilometer.storage.impl_test import require_map_reduce
from ceilometer.storage.impl_mongodb import require_map_reduce
class TestSumProjectVolume(tests_api.TestBase):

2
tests/api/v1/test_sum_resource_volume.py
View File

@ -27,7 +27,7 @@ from ceilometer.collector import meter
from ceilometer import counter
from ceilometer.tests import api as tests_api
from ceilometer.storage.impl_test import require_map_reduce
from ceilometer.storage.impl_mongodb import require_map_reduce
class TestSumResourceVolume(tests_api.TestBase):

4
tests/api/v2/test_compute_duration_by_resource.py
View File

@ -22,7 +22,7 @@ import datetime
import logging
from ceilometer.openstack.common import timeutils
from ceilometer.storage import impl_test
from ceilometer.storage import impl_mongodb
from .base import FunctionalTest
LOG = logging.getLogger(__name__)
@ -50,7 +50,7 @@ class TestComputeDurationByResource(FunctionalTest):
self.late2 = datetime.datetime(2012, 8, 29, 19, 0)
def _stub_interval_func(self, func):
self.stubs.Set(impl_test.TestConnection,
self.stubs.Set(impl_mongodb.Connection,
'get_meter_statistics',
func)

2
tests/api/v2/test_max_project_volume.py
View File

@ -25,7 +25,7 @@ from oslo.config import cfg
from ceilometer.collector import meter
from ceilometer import counter
from ceilometer.storage.impl_test import require_map_reduce
from ceilometer.storage.impl_mongodb import require_map_reduce
from .base import FunctionalTest

2
tests/api/v2/test_max_resource_volume.py
View File

@ -26,7 +26,7 @@ from ceilometer.collector import meter
from ceilometer import counter
from .base import FunctionalTest
from ceilometer.storage.impl_test import require_map_reduce
from ceilometer.storage.impl_mongodb import require_map_reduce
class TestMaxResourceVolume(FunctionalTest):

2
tests/api/v2/test_sum_project_volume.py
View File

@ -26,7 +26,7 @@ from ceilometer.collector import meter
from ceilometer import counter
from .base import FunctionalTest
from ceilometer.storage.impl_test import require_map_reduce
from ceilometer.storage.impl_mongodb import require_map_reduce
class TestSumProjectVolume(FunctionalTest):

2
tests/api/v2/test_sum_resource_volume.py
View File

@ -26,7 +26,7 @@ from ceilometer.collector import meter
from ceilometer import counter
from .base import FunctionalTest
from ceilometer.storage.impl_test import require_map_reduce
from ceilometer.storage.impl_mongodb import require_map_reduce
class TestSumResourceVolume(FunctionalTest):

4
tests/storage/test_impl_mongodb.py
View File

@ -53,11 +53,11 @@ from tests.storage import base
from ceilometer.collector import meter
from ceilometer import counter
from ceilometer.storage.impl_test import require_map_reduce
from ceilometer.storage.impl_mongodb import require_map_reduce
class MongoDBEngineTestBase(base.DBTestBase):
database_connection = 'test://'
database_connection = 'mongodb://__test__'
class IndexTest(MongoDBEngineTestBase):