openstack/trove
Code Issues Proposed changes

Unittests patch 'CONF.datastore_manager'

Browse Source 
Unittests do not have 'CONF.datastore_manager' set (None).

As the result all datastore manager/app tests that access
configuration by the manager name get the default 'mysql' values.
Apart from not testing the production configuration this
also makes it difficult to test dynamically looked up
datastore-specific properties which do not exist in the mysql
group at all.

This patch set adds a helper method for patching any given
configuration property and also adds a simple interface for
patching the manager name to the base TestCase.

All existing manager/app tests were updated to properly patch the manager
property.
The base test patches the manager name to 'mysql' so that
all unittests run with this manager unless explicitly overriden.

It's now safe to remove the manager parameter from the
'cfg.get_configuration_property()' function. It was there
to work around this issue in unittests.

Change-Id: I903a0fb1ede8ad6614d2b5a8fc1a256f74412496
This commit is contained in:
Petr Malik 2016-04-20 21:36:27 -04:00
parent 04bcdfe9bd
commit c45eb67018
18 changed files with 105 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
trove/common/cfg.py
View File

@ -26,6 +26,7 @@ from osprofiler import opts as profiler
from trove.version import version_info as version
LOG = logging.getLogger(__name__)
UNKNOWN_SERVICE_ID = 'unknown-service-id-error'
path_opts = [
@ -1439,35 +1440,35 @@ def parse_args(argv, default_config_files=None):
default_config_files=default_config_files)
def get_ignored_dbs(manager=None):
def get_ignored_dbs():
try:
return get_configuration_property('ignore_dbs', manager=manager)
return get_configuration_property('ignore_dbs')
except NoSuchOptError:
return []
def get_ignored_users(manager=None):
def get_ignored_users():
try:
return get_configuration_property('ignore_users', manager=manager)
return get_configuration_property('ignore_users')
except NoSuchOptError:
return []
def get_configuration_property(property_name, manager=None):
def get_configuration_property(property_name):
"""
Get a configuration property.
Try to get it from the datastore-specific section first.
If it is not available, retrieve it from the DEFAULT section.
"""
# TODO(pmalik): Note that the unit and fake-integration tests
# do not define 'CONF.datastore_manager'. *MySQL* options will
# be loaded unless the caller passes a manager name explicitly.
#
# Once the tests are fixed this conditional expression should be removed
# and the proper value should always be either loaded from
# 'CONF.datastore_manager' or passed-in by the caller.
datastore_manager = manager or CONF.datastore_manager or 'mysql'
# Fake-integration tests do not define 'CONF.datastore_manager'.
# *MySQL* options will
# be loaded. This should never occur in a production environment.
datastore_manager = CONF.datastore_manager
if not datastore_manager:
datastore_manager = 'mysql'
LOG.warning(_("Manager name ('datastore_manager') not defined, "
"using '%s' options instead.") % datastore_manager)
try:
return CONF.get(datastore_manager).get(property_name)

6
trove/guestagent/datastore/experimental/couchdb/service.py
View File

@ -35,7 +35,6 @@ from trove.guestagent import pkg
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
packager = pkg.Package()
MANAGER = CONF.datastore_manager if CONF.datastore_manager else 'couchdb'
COUCHDB_LIB_DIR = "/var/lib/couchdb"
COUCHDB_LOG_DIR = "/var/log/couchdb"
@ -207,7 +206,7 @@ class CouchDBAdmin(object):
return type(self).admin_user
def _is_modifiable_user(self, name):
if name in cfg.get_ignored_users(manager=MANAGER):
if name in cfg.get_ignored_users():
return False
elif name == system.COUCHDB_ADMIN_NAME:
return False
@ -523,7 +522,7 @@ class CouchDBAdmin(object):
'admin_password': self._admin_user().password},
shell=True)
dbnames_list = eval(out)
for hidden in cfg.get_ignored_dbs(manager=MANAGER):
for hidden in cfg.get_ignored_dbs():
if hidden in dbnames_list:
dbnames_list.remove(hidden)
return dbnames_list
@ -560,6 +559,7 @@ class CouchDBAdmin(object):
class CouchDBCredentials(object):
"""Handles storing/retrieving credentials. Stored as json in files"""
def __init__(self, username=None, password=None):
self.username = username
self.password = password

2
trove/guestagent/datastore/experimental/db2/service.py
View File

@ -353,7 +353,7 @@ class DB2Admin(object):
user = item.split() if item != "" else None
LOG.debug("user = %r" % (user))
if (user is not None
and (user[0] not in cfg.get_ignored_users(manager='db2')
and (user[0] not in cfg.get_ignored_users()
and user[1] == 'Y')):
userlist.append(user[0])
result = iter(userlist)

4
trove/guestagent/datastore/experimental/mongodb/service.py
View File

@ -480,7 +480,7 @@ class MongoDBAdmin(object):
return type(self).admin_user
def _is_modifiable_user(self, name):
if ((name in cfg.get_ignored_users(manager=MANAGER)) or
if ((name in cfg.get_ignored_users()) or
name == system.MONGO_ADMIN_NAME):
return False
return True
@ -723,7 +723,7 @@ class MongoDBAdmin(object):
def list_databases(self, limit=None, marker=None, include_marker=False):
"""Lists the databases."""
db_names = self.list_database_names()
for hidden in cfg.get_ignored_dbs(manager=MANAGER):
for hidden in cfg.get_ignored_dbs():
if hidden in db_names:
db_names.remove(hidden)
databases = [models.MongoDBSchema(db_name).serialize()

3
trove/guestagent/datastore/experimental/postgresql/service/database.py
View File

@ -94,8 +94,7 @@ class PgSqlDatabase(object):
[{"_name": "", "_character_set": "", "_collate": ""}, ...]
"""
results = pgutil.query(
pgutil.DatabaseQuery.list(ignore=cfg.get_ignored_dbs(
manager='postgresql')),
pgutil.DatabaseQuery.list(ignore=cfg.get_ignored_dbs()),
timeout=30,
)
# Convert results to dictionaries.

3
trove/guestagent/datastore/experimental/postgresql/service/users.py
View File

@ -115,8 +115,7 @@ class PgSqlUsers(PgSqlAccess):
"_databases": [{"_name": ""}, ...]}, ...]
"""
results = pgutil.query(
pgutil.UserQuery.list(ignore=cfg.get_ignored_users(
manager='postgresql')),
pgutil.UserQuery.list(ignore=cfg.get_ignored_users()),
timeout=30,
)
# Convert results into dictionaries.

6
trove/tests/unittests/guestagent/test_cassandra_manager.py
View File

@ -38,10 +38,12 @@ from trove.guestagent.datastore.experimental.cassandra import (
from trove.guestagent.db import models
from trove.guestagent import pkg as pkg
from trove.guestagent import volume
from trove.tests.unittests.guestagent.test_datastore_manager import \
DatastoreManagerTest
from trove.tests.unittests import trove_testtools
class GuestAgentCassandraDBManagerTest(trove_testtools.TestCase):
class GuestAgentCassandraDBManagerTest(DatastoreManagerTest):
__MOUNT_POINT = '/var/lib/cassandra'
@ -74,7 +76,7 @@ class GuestAgentCassandraDBManagerTest(trove_testtools.TestCase):
@patch.object(ImportOverrideStrategy, '_initialize_import_directory')
@patch('trove.guestagent.datastore.experimental.cassandra.service.LOG')
def setUp(self, *args, **kwargs):
super(GuestAgentCassandraDBManagerTest, self).setUp()
super(GuestAgentCassandraDBManagerTest, self).setUp('cassandra')
conn_patcher = patch.multiple(cass_service.CassandraConnection,
_connect=DEFAULT,

8
trove/tests/unittests/guestagent/test_couchbase_manager.py
View File

@ -30,14 +30,14 @@ from trove.guestagent.datastore.experimental.couchbase import (
from trove.guestagent.datastore.experimental.couchbase import (
service as couch_service)
from trove.guestagent import volume
from trove.tests.unittests import trove_testtools
from trove.tests.unittests.guestagent.test_datastore_manager import \
DatastoreManagerTest
class GuestAgentCouchbaseManagerTest(trove_testtools.TestCase):
class GuestAgentCouchbaseManagerTest(DatastoreManagerTest):
def setUp(self):
super(GuestAgentCouchbaseManagerTest, self).setUp()
self.context = trove_testtools.TroveTestContext(self)
super(GuestAgentCouchbaseManagerTest, self).setUp('couchbase')
self.manager = couch_manager.Manager()
self.packages = 'couchbase-server'
app_patcher = patch.multiple(

8
trove/tests/unittests/guestagent/test_couchdb_manager.py
View File

@ -27,13 +27,14 @@ from trove.guestagent.datastore.experimental.couchdb import (
service as couchdb_service)
from trove.guestagent import pkg as pkg
from trove.guestagent import volume
from trove.tests.unittests import trove_testtools
from trove.tests.unittests.guestagent.test_datastore_manager import \
DatastoreManagerTest
class GuestAgentCouchDBManagerTest(trove_testtools.TestCase):
class GuestAgentCouchDBManagerTest(DatastoreManagerTest):
def setUp(self):
super(GuestAgentCouchDBManagerTest, self).setUp()
super(GuestAgentCouchDBManagerTest, self).setUp('couchdb')
self.real_status = couchdb_service.CouchDBAppStatus.set_status
class FakeInstanceServiceStatus(object):
@ -44,7 +45,6 @@ class GuestAgentCouchDBManagerTest(trove_testtools.TestCase):
couchdb_service.CouchDBAppStatus.set_status = MagicMock(
return_value=FakeInstanceServiceStatus())
self.context = trove_testtools.TroveTestContext(self)
self.manager = couchdb_manager.Manager()
self.pkg = couchdb_service.packager
self.real_db_app_status = couchdb_service.CouchDBAppStatus

24
trove/tests/unittests/guestagent/test_datastore_manager.py Normal file
View File

@ -0,0 +1,24 @@
# Copyright 2016 Tesora Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from trove.tests.unittests import trove_testtools
class DatastoreManagerTest(trove_testtools.TestCase):
def setUp(self, manager_name):
super(DatastoreManagerTest, self).setUp()
self.patch_datastore_manager(manager_name)
self.context = trove_testtools.TroveTestContext(self)

8
trove/tests/unittests/guestagent/test_db2_manager.py
View File

@ -24,13 +24,14 @@ from trove.guestagent.datastore.experimental.db2 import (
service as db2_service)
from trove.guestagent import pkg as pkg
from trove.guestagent import volume
from trove.tests.unittests import trove_testtools
from trove.tests.unittests.guestagent.test_datastore_manager import \
DatastoreManagerTest
class GuestAgentDB2ManagerTest(trove_testtools.TestCase):
class GuestAgentDB2ManagerTest(DatastoreManagerTest):
def setUp(self):
super(GuestAgentDB2ManagerTest, self).setUp()
super(GuestAgentDB2ManagerTest, self).setUp('db2')
self.real_status = db2_service.DB2AppStatus.set_status
class FakeInstanceServiceStatus(object):
@ -41,7 +42,6 @@ class GuestAgentDB2ManagerTest(trove_testtools.TestCase):
db2_service.DB2AppStatus.set_status = MagicMock(
return_value=FakeInstanceServiceStatus())
self.context = trove_testtools.TroveTestContext(self)
self.manager = db2_manager.Manager()
self.real_db_app_status = db2_service.DB2AppStatus
self.origin_format = volume.VolumeDevice.format

15
trove/tests/unittests/guestagent/test_dbaas.py
View File

@ -297,8 +297,9 @@ class BaseAppTest(object):
class AppTestCase(trove_testtools.TestCase):
def setUp(self, fake_id):
def setUp(self, fake_id, manager_name):
super(BaseAppTest.AppTestCase, self).setUp()
self.patch_datastore_manager(manager_name)
self.FAKE_ID = fake_id
InstanceServiceStatus.create(
instance_id=self.FAKE_ID,
@ -2315,7 +2316,7 @@ class MySqlAppStatusTest(trove_testtools.TestCase):
class TestRedisApp(BaseAppTest.AppTestCase):
def setUp(self):
super(TestRedisApp, self).setUp(str(uuid4()))
super(TestRedisApp, self).setUp(str(uuid4()), 'redis')
self.orig_os_path_eu = os.path.expanduser
os.path.expanduser = Mock(return_value='/tmp/.file')
@ -2389,7 +2390,7 @@ class CassandraDBAppTest(BaseAppTest.AppTestCase):
@patch.object(ImportOverrideStrategy, '_initialize_import_directory')
@patch('trove.guestagent.datastore.experimental.cassandra.service.LOG')
def setUp(self, mock_logging, _):
super(CassandraDBAppTest, self).setUp(str(uuid4()))
super(CassandraDBAppTest, self).setUp(str(uuid4()), 'cassandra')
self.sleep = time.sleep
self.orig_time_time = time.time
self.pkg_version = cass_service.packager.pkg_version
@ -2470,7 +2471,7 @@ class CouchbaseAppTest(BaseAppTest.AppTestCase):
}
def setUp(self):
super(CouchbaseAppTest, self).setUp(str(uuid4()))
super(CouchbaseAppTest, self).setUp(str(uuid4()), 'couchbase')
self.orig_utils_execute_with_timeout = (
couchservice.utils.execute_with_timeout)
self.orig_time_sleep = time.sleep
@ -2538,7 +2539,7 @@ class CouchDBAppTest(BaseAppTest.AppTestCase):
}
def setUp(self):
super(CouchDBAppTest, self).setUp(str(uuid4()))
super(CouchDBAppTest, self).setUp(str(uuid4()), 'couchdb')
self.orig_utils_execute_with_timeout = (
couchdb_service.utils.execute_with_timeout)
self.orig_time_sleep = time.sleep
@ -2603,7 +2604,7 @@ class MongoDBAppTest(BaseAppTest.AppTestCase):
@patch.object(ImportOverrideStrategy, '_initialize_import_directory')
def setUp(self, _):
super(MongoDBAppTest, self).setUp(str(uuid4()))
super(MongoDBAppTest, self).setUp(str(uuid4()), 'mongodb')
self.orig_utils_execute_with_timeout = (mongo_service.
utils.execute_with_timeout)
self.orig_time_sleep = time.sleep
@ -3713,7 +3714,7 @@ class PostgresAppTest(BaseAppTest.AppTestCase):
@patch.object(pg_config.PgSqlConfig, '_find_config_file', return_value='')
def setUp(self, _):
super(PostgresAppTest, self).setUp(str(uuid4()))
super(PostgresAppTest, self).setUp(str(uuid4()), 'postgresql')
self.orig_time_sleep = time.sleep
self.orig_time_time = time.time
time.sleep = Mock()

9
trove/tests/unittests/guestagent/test_mariadb_manager.py
View File

@ -14,22 +14,21 @@
from mock import MagicMock
from mock import patch
import testtools
from trove.common.context import TroveContext
from trove.guestagent.datastore.experimental.mariadb import (
manager as mariadb_manager)
from trove.guestagent.datastore.experimental.mariadb import (
service as mariadb_service)
from trove.guestagent.datastore.mysql_common import service as mysql_service
from trove.tests.unittests.guestagent.test_datastore_manager import \
DatastoreManagerTest
class GuestAgentManagerTest(testtools.TestCase):
class GuestAgentManagerTest(DatastoreManagerTest):
def setUp(self):
super(GuestAgentManagerTest, self).setUp()
super(GuestAgentManagerTest, self).setUp('mariadb')
self.manager = mariadb_manager.Manager()
self.context = TroveContext()
patcher_rs = patch(
'trove.guestagent.strategies.replication.get_instance')
patcher_rs.start()

8
trove/tests/unittests/guestagent/test_mongodb_manager.py
View File

@ -22,15 +22,15 @@ import trove.guestagent.datastore.experimental.mongodb.manager as manager
import trove.guestagent.datastore.experimental.mongodb.service as service
import trove.guestagent.db.models as models
import trove.guestagent.volume as volume
import trove.tests.unittests.trove_testtools as trove_testtools
from trove.tests.unittests.guestagent.test_datastore_manager import \
DatastoreManagerTest
class GuestAgentMongoDBManagerTest(trove_testtools.TestCase):
class GuestAgentMongoDBManagerTest(DatastoreManagerTest):
@mock.patch.object(ImportOverrideStrategy, '_initialize_import_directory')
def setUp(self, _):
super(GuestAgentMongoDBManagerTest, self).setUp()
self.context = trove_testtools.TroveTestContext(self)
super(GuestAgentMongoDBManagerTest, self).setUp('mongodb')
self.manager = manager.Manager()
self.execute_with_timeout_patch = mock.patch.object(

6
trove/tests/unittests/guestagent/test_mysql_manager.py
View File

@ -33,13 +33,15 @@ from trove.guestagent import dbaas as base_dbaas
from trove.guestagent import pkg as pkg
from trove.guestagent import volume
from trove.guestagent.volume import VolumeDevice
from trove.tests.unittests.guestagent.test_datastore_manager import \
DatastoreManagerTest
from trove.tests.unittests import trove_testtools
class GuestAgentManagerTest(trove_testtools.TestCase):
class GuestAgentManagerTest(DatastoreManagerTest):
def setUp(self):
super(GuestAgentManagerTest, self).setUp()
super(GuestAgentManagerTest, self).setUp('mysql')
self.context = trove_testtools.TroveTestContext(self)
self.replication_strategy = 'MysqlGTIDReplication'
self.patch_rs = patch(

8
trove/tests/unittests/guestagent/test_redis_manager.py
View File

@ -23,19 +23,19 @@ from trove.guestagent.datastore.experimental.redis import (
from trove.guestagent.datastore.experimental.redis.manager import (
Manager as RedisManager)
from trove.guestagent.volume import VolumeDevice
from trove.tests.unittests import trove_testtools
from trove.tests.unittests.guestagent.test_datastore_manager import \
DatastoreManagerTest
class RedisGuestAgentManagerTest(trove_testtools.TestCase):
class RedisGuestAgentManagerTest(DatastoreManagerTest):
@patch.object(redis_service.RedisApp, '_build_admin_client')
@patch.object(ImportOverrideStrategy, '_initialize_import_directory')
def setUp(self, *args, **kwargs):
super(RedisGuestAgentManagerTest, self).setUp()
super(RedisGuestAgentManagerTest, self).setUp('redis')
self.patch_ope = patch('os.path.expanduser')
self.mock_ope = self.patch_ope.start()
self.addCleanup(self.patch_ope.stop)
self.context = trove_testtools.TroveTestContext(self)
self.replication_strategy = 'RedisSyncReplication'
self.patch_rs = patch(
'trove.guestagent.strategies.replication.get_strategy',

8
trove/tests/unittests/guestagent/test_vertica_manager.py
View File

@ -30,17 +30,17 @@ from trove.guestagent.datastore.experimental.vertica import system
from trove.guestagent import dbaas
from trove.guestagent import volume
from trove.guestagent.volume import VolumeDevice
from trove.tests.unittests import trove_testtools
from trove.tests.unittests.guestagent.test_datastore_manager import \
DatastoreManagerTest
class GuestAgentManagerTest(trove_testtools.TestCase):
class GuestAgentManagerTest(DatastoreManagerTest):
@patch.object(ImportOverrideStrategy, '_initialize_import_directory')
@patch.multiple(operating_system, exists=DEFAULT, write_file=DEFAULT,
chown=DEFAULT, chmod=DEFAULT)
def setUp(self, *args, **kwargs):
super(GuestAgentManagerTest, self).setUp()
self.context = trove_testtools.TroveTestContext(self)
super(GuestAgentManagerTest, self).setUp('vertica')
self.manager = Manager()
self.origin_format = volume.VolumeDevice.format
self.origin_migrate_data = volume.VolumeDevice.migrate_data

14
trove/tests/unittests/trove_testtools.py
View File

@ -20,6 +20,7 @@ import os
import sys
import testtools
from trove.common import cfg
from trove.common.context import TroveContext
from trove.common.notification import DBaaSAPINotification
from trove.tests import root_logger
@ -98,6 +99,9 @@ class TestCase(testtools.TestCase):
super(TestCase, self).setUp()
root_logger.DefaultRootHandler.set_info(self.id())
# Default manager used by all unittsest unless explicitly overriden.
self.patch_datastore_manager('mysql')
def tearDown(self):
# yes, this is gross and not thread aware.
# but the only way to make it thread aware would require that
@ -161,3 +165,13 @@ class TestCase(testtools.TestCase):
@classmethod
def _get_loaded_modules(cls):
return {name: obj for name, obj in sys.modules.items() if obj}
def patch_datastore_manager(self, manager_name):
return self.patch_conf_property('datastore_manager', manager_name)
def patch_conf_property(self, property_name, value):
conf_patcher = mock.patch.object(
cfg.CONF, property_name,
new_callable=mock.PropertyMock(return_value=value))
self.addCleanup(conf_patcher.stop)
return conf_patcher.start()