diff --git a/keystone/common/cache/__init__.py b/keystone/common/cache/__init__.py deleted file mode 100644 index 49502399..00000000 --- a/keystone/common/cache/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2013 Metacloud -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from keystone.common.cache.core import * # noqa diff --git a/keystone/tests/unit/test_cache.py b/keystone/tests/unit/test_cache.py deleted file mode 100644 index 5a778a07..00000000 --- a/keystone/tests/unit/test_cache.py +++ /dev/null @@ -1,322 +0,0 @@ -# Copyright 2013 Metacloud -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import copy -import time -import uuid - -from dogpile.cache import api -from dogpile.cache import proxy -import mock -from oslo_config import cfg - -from keystone.common import cache -from keystone import exception -from keystone.tests import unit as tests - - -CONF = cfg.CONF -NO_VALUE = api.NO_VALUE - - -def _copy_value(value): - if value is not NO_VALUE: - value = copy.deepcopy(value) - return value - - -# NOTE(morganfainberg): WARNING - It is not recommended to use the Memory -# backend for dogpile.cache in a real deployment under any circumstances. The -# backend does no cleanup of expired values and therefore will leak memory. The -# backend is not implemented in a way to share data across processes (e.g. -# Keystone in HTTPD. This proxy is a hack to get around the lack of isolation -# of values in memory. Currently it blindly stores and retrieves the values -# from the cache, and modifications to dicts/lists/etc returned can result in -# changes to the cached values. In short, do not use the dogpile.cache.memory -# backend unless you are running tests or expecting odd/strange results. -class CacheIsolatingProxy(proxy.ProxyBackend): - """Proxy that forces a memory copy of stored values. - The default in-memory cache-region does not perform a copy on values it - is meant to cache. Therefore if the value is modified after set or after - get, the cached value also is modified. This proxy does a copy as the last - thing before storing data. - """ - def get(self, key): - return _copy_value(self.proxied.get(key)) - - def set(self, key, value): - self.proxied.set(key, _copy_value(value)) - - -class TestProxy(proxy.ProxyBackend): - def get(self, key): - value = _copy_value(self.proxied.get(key)) - if value is not NO_VALUE: - if isinstance(value[0], TestProxyValue): - value[0].cached = True - return value - - -class TestProxyValue(object): - def __init__(self, value): - self.value = value - self.cached = False - - -class CacheRegionTest(tests.TestCase): - - def setUp(self): - super(CacheRegionTest, self).setUp() - self.region = cache.make_region() - cache.configure_cache_region(self.region) - self.region.wrap(TestProxy) - self.test_value = TestProxyValue('Decorator Test') - - def _add_test_caching_option(self): - self.config_fixture.register_opt( - cfg.BoolOpt('caching', default=True), group='cache') - - def _get_cacheable_function(self): - with mock.patch.object(cache.REGION, 'cache_on_arguments', - self.region.cache_on_arguments): - memoize = cache.get_memoization_decorator(section='cache') - - @memoize - def cacheable_function(value): - return value - - return cacheable_function - - def test_region_built_with_proxy_direct_cache_test(self): - # Verify cache regions are properly built with proxies. - test_value = TestProxyValue('Direct Cache Test') - self.region.set('cache_test', test_value) - cached_value = self.region.get('cache_test') - self.assertTrue(cached_value.cached) - - def test_cache_region_no_error_multiple_config(self): - # Verify configuring the CacheRegion again doesn't error. - cache.configure_cache_region(self.region) - cache.configure_cache_region(self.region) - - def _get_cache_fallthrough_fn(self, cache_time): - with mock.patch.object(cache.REGION, 'cache_on_arguments', - self.region.cache_on_arguments): - memoize = cache.get_memoization_decorator( - section='cache', - expiration_section='assignment') - - class _test_obj(object): - def __init__(self, value): - self.test_value = value - - @memoize - def get_test_value(self): - return self.test_value - - def _do_test(value): - - test_obj = _test_obj(value) - - # Ensure the value has been cached - test_obj.get_test_value() - # Get the now cached value - cached_value = test_obj.get_test_value() - self.assertTrue(cached_value.cached) - self.assertEqual(value.value, cached_value.value) - self.assertEqual(cached_value.value, test_obj.test_value.value) - # Change the underlying value on the test object. - test_obj.test_value = TestProxyValue(uuid.uuid4().hex) - self.assertEqual(cached_value.value, - test_obj.get_test_value().value) - # override the system time to ensure the non-cached new value - # is returned - new_time = time.time() + (cache_time * 2) - with mock.patch.object(time, 'time', - return_value=new_time): - overriden_cache_value = test_obj.get_test_value() - self.assertNotEqual(cached_value.value, - overriden_cache_value.value) - self.assertEqual(test_obj.test_value.value, - overriden_cache_value.value) - - return _do_test - - def test_cache_no_fallthrough_expiration_time_fn(self): - # Since we do not re-configure the cache region, for ease of testing - # this value is set the same as the expiration_time default in the - # [cache] section - cache_time = 600 - expiration_time = cache.get_expiration_time_fn('role') - do_test = self._get_cache_fallthrough_fn(cache_time) - # Run the test with the assignment cache_time value - self.config_fixture.config(cache_time=cache_time, - group='role') - test_value = TestProxyValue(uuid.uuid4().hex) - self.assertEqual(cache_time, expiration_time()) - do_test(value=test_value) - - def test_cache_fallthrough_expiration_time_fn(self): - # Since we do not re-configure the cache region, for ease of testing - # this value is set the same as the expiration_time default in the - # [cache] section - cache_time = 599 - expiration_time = cache.get_expiration_time_fn('role') - do_test = self._get_cache_fallthrough_fn(cache_time) - # Run the test with the assignment cache_time value set to None and - # the global value set. - self.config_fixture.config(cache_time=None, group='role') - test_value = TestProxyValue(uuid.uuid4().hex) - self.assertIsNone(expiration_time()) - do_test(value=test_value) - - def test_should_cache_fn_global_cache_enabled(self): - # Verify should_cache_fn generates a sane function for subsystem and - # functions as expected with caching globally enabled. - cacheable_function = self._get_cacheable_function() - - self.config_fixture.config(group='cache', enabled=True) - cacheable_function(self.test_value) - cached_value = cacheable_function(self.test_value) - self.assertTrue(cached_value.cached) - - def test_should_cache_fn_global_cache_disabled(self): - # Verify should_cache_fn generates a sane function for subsystem and - # functions as expected with caching globally disabled. - cacheable_function = self._get_cacheable_function() - - self.config_fixture.config(group='cache', enabled=False) - cacheable_function(self.test_value) - cached_value = cacheable_function(self.test_value) - self.assertFalse(cached_value.cached) - - def test_should_cache_fn_global_cache_disabled_section_cache_enabled(self): - # Verify should_cache_fn generates a sane function for subsystem and - # functions as expected with caching globally disabled and the specific - # section caching enabled. - cacheable_function = self._get_cacheable_function() - - self._add_test_caching_option() - self.config_fixture.config(group='cache', enabled=False) - self.config_fixture.config(group='cache', caching=True) - - cacheable_function(self.test_value) - cached_value = cacheable_function(self.test_value) - self.assertFalse(cached_value.cached) - - def test_should_cache_fn_global_cache_enabled_section_cache_disabled(self): - # Verify should_cache_fn generates a sane function for subsystem and - # functions as expected with caching globally enabled and the specific - # section caching disabled. - cacheable_function = self._get_cacheable_function() - - self._add_test_caching_option() - self.config_fixture.config(group='cache', enabled=True) - self.config_fixture.config(group='cache', caching=False) - - cacheable_function(self.test_value) - cached_value = cacheable_function(self.test_value) - self.assertFalse(cached_value.cached) - - def test_should_cache_fn_global_cache_enabled_section_cache_enabled(self): - # Verify should_cache_fn generates a sane function for subsystem and - # functions as expected with caching globally enabled and the specific - # section caching enabled. - cacheable_function = self._get_cacheable_function() - - self._add_test_caching_option() - self.config_fixture.config(group='cache', enabled=True) - self.config_fixture.config(group='cache', caching=True) - - cacheable_function(self.test_value) - cached_value = cacheable_function(self.test_value) - self.assertTrue(cached_value.cached) - - def test_cache_dictionary_config_builder(self): - """Validate we build a sane dogpile.cache dictionary config.""" - self.config_fixture.config(group='cache', - config_prefix='test_prefix', - backend='some_test_backend', - expiration_time=86400, - backend_argument=['arg1:test', - 'arg2:test:test', - 'arg3.invalid']) - - config_dict = cache.build_cache_config() - self.assertEqual( - CONF.cache.backend, config_dict['test_prefix.backend']) - self.assertEqual( - CONF.cache.expiration_time, - config_dict['test_prefix.expiration_time']) - self.assertEqual('test', config_dict['test_prefix.arguments.arg1']) - self.assertEqual('test:test', - config_dict['test_prefix.arguments.arg2']) - self.assertNotIn('test_prefix.arguments.arg3', config_dict) - - def test_cache_debug_proxy(self): - single_value = 'Test Value' - single_key = 'testkey' - multi_values = {'key1': 1, 'key2': 2, 'key3': 3} - - self.region.set(single_key, single_value) - self.assertEqual(single_value, self.region.get(single_key)) - - self.region.delete(single_key) - self.assertEqual(NO_VALUE, self.region.get(single_key)) - - self.region.set_multi(multi_values) - cached_values = self.region.get_multi(multi_values.keys()) - for value in multi_values.values(): - self.assertIn(value, cached_values) - self.assertEqual(len(multi_values.values()), len(cached_values)) - - self.region.delete_multi(multi_values.keys()) - for value in self.region.get_multi(multi_values.keys()): - self.assertEqual(NO_VALUE, value) - - def test_configure_non_region_object_raises_error(self): - self.assertRaises(exception.ValidationError, - cache.configure_cache_region, - "bogus") - - -class CacheNoopBackendTest(tests.TestCase): - - def setUp(self): - super(CacheNoopBackendTest, self).setUp() - self.region = cache.make_region() - cache.configure_cache_region(self.region) - - def config_overrides(self): - super(CacheNoopBackendTest, self).config_overrides() - self.config_fixture.config(group='cache', - backend='keystone.common.cache.noop') - - def test_noop_backend(self): - single_value = 'Test Value' - single_key = 'testkey' - multi_values = {'key1': 1, 'key2': 2, 'key3': 3} - - self.region.set(single_key, single_value) - self.assertEqual(NO_VALUE, self.region.get(single_key)) - - self.region.set_multi(multi_values) - cached_values = self.region.get_multi(multi_values.keys()) - self.assertEqual(len(cached_values), len(multi_values.values())) - for value in cached_values: - self.assertEqual(NO_VALUE, value) - - # Delete should not raise exceptions - self.region.delete(single_key) - self.region.delete_multi(multi_values.keys()) diff --git a/oslo_cache/_i18n.py b/oslo_cache/_i18n.py new file mode 100644 index 00000000..92659914 --- /dev/null +++ b/oslo_cache/_i18n.py @@ -0,0 +1,35 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""oslo.i18n integration module. + +See http://docs.openstack.org/developer/oslo.i18n/usage.html + +""" + +import oslo_i18n + + +_translators = oslo_i18n.TranslatorFactory(domain='oslo.versionedobjects') + +# The primary translation function using the well-known name "_" +_ = _translators.primary + +# Translators for log levels. +# +# The abbreviated names are meant to reflect the usual use of a short +# name like '_'. The "L" is for "log" and the other letter comes from +# the level. +_LI = _translators.log_info +_LW = _translators.log_warning +_LE = _translators.log_error +_LC = _translators.log_critical diff --git a/keystone/common/cache/_memcache_pool.py b/oslo_cache/_memcache_pool.py similarity index 99% rename from keystone/common/cache/_memcache_pool.py rename to oslo_cache/_memcache_pool.py index 2bfcc3bb..f531f1d2 100644 --- a/keystone/common/cache/_memcache_pool.py +++ b/oslo_cache/_memcache_pool.py @@ -29,8 +29,8 @@ import memcache from oslo_log import log from six.moves import queue, zip -from keystone import exception -from keystone.i18n import _ +from oslo_cache._i18n import _ +from oslo_cache import exception LOG = log.getLogger(__name__) diff --git a/oslo_cache/_opts.py b/oslo_cache/_opts.py new file mode 100644 index 00000000..677a4776 --- /dev/null +++ b/oslo_cache/_opts.py @@ -0,0 +1,131 @@ +# Copyright 2012 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + + +FILE_OPTIONS = { + 'cache': [ + cfg.StrOpt('config_prefix', default='cache.keystone', + help='Prefix for building the configuration dictionary ' + 'for the cache region. This should not need to be ' + 'changed unless there is another dogpile.cache ' + 'region with the same configuration name.'), + cfg.IntOpt('expiration_time', default=600, + help='Default TTL, in seconds, for any cached item in ' + 'the dogpile.cache region. This applies to any ' + 'cached method that doesn\'t have an explicit ' + 'cache expiration time defined for it.'), + # NOTE(morganfainberg): the dogpile.cache.memory acceptable in devstack + # and other such single-process/thread deployments. Running + # dogpile.cache.memory in any other configuration has the same pitfalls + # as the KVS token backend. It is recommended that either Redis or + # Memcached are used as the dogpile backend for real workloads. To + # prevent issues with the memory cache ending up in "production" + # unintentionally, we register a no-op as the keystone default caching + # backend. + cfg.StrOpt('backend', default='oslo_cache.noop', + help='Dogpile.cache backend module. It is recommended ' + 'that Memcache with pooling ' + '(keystone.cache.memcache_pool) or Redis ' + '(dogpile.cache.redis) be used in production ' + 'deployments. Small workloads (single process) ' + 'like devstack can use the dogpile.cache.memory ' + 'backend.'), + cfg.MultiStrOpt('backend_argument', default=[], secret=True, + help='Arguments supplied to the backend module. ' + 'Specify this option once per argument to be ' + 'passed to the dogpile.cache backend. Example ' + 'format: ":".'), + cfg.ListOpt('proxies', default=[], + help='Proxy classes to import that will affect the way ' + 'the dogpile.cache backend functions. See the ' + 'dogpile.cache documentation on ' + 'changing-backend-behavior.'), + cfg.BoolOpt('enabled', default=False, + help='Global toggle for all caching using the ' + 'should_cache_fn mechanism.'), + cfg.BoolOpt('debug_cache_backend', default=False, + help='Extra debugging from the cache backend (cache ' + 'keys, get/set/delete/etc calls). This is only ' + 'really useful if you need to see the specific ' + 'cache-backend get/set/delete calls with the ' + 'keys/values. Typically this should be left set ' + 'to false.'), + cfg.ListOpt('memcache_servers', default=['localhost:11211'], + help='Memcache servers in the format of "host:port".' + ' (dogpile.cache.memcache and keystone.cache.memcache_pool' + ' backends only).'), + cfg.IntOpt('memcache_dead_retry', + default=5 * 60, + help='Number of seconds memcached server is considered dead' + ' before it is tried again. (dogpile.cache.memcache and' + ' keystone.cache.memcache_pool backends only).'), + cfg.IntOpt('memcache_socket_timeout', + default=3, + help='Timeout in seconds for every call to a server.' + ' (dogpile.cache.memcache and keystone.cache.memcache_pool' + ' backends only).'), + cfg.IntOpt('memcache_pool_maxsize', + default=10, + help='Max total number of open connections to every' + ' memcached server. (keystone.cache.memcache_pool backend' + ' only).'), + cfg.IntOpt('memcache_pool_unused_timeout', + default=60, + help='Number of seconds a connection to memcached is held' + ' unused in the pool before it is closed.' + ' (keystone.cache.memcache_pool backend only).'), + cfg.IntOpt('memcache_pool_connection_get_timeout', + default=10, + help='Number of seconds that an operation will wait to get ' + 'a memcache client connection.'), + ], +} + + +CONF = cfg.CONF + + +def configure(conf=None): + if conf is None: + conf = CONF + + for section in FILE_OPTIONS: + for option in FILE_OPTIONS[section]: + conf.register_opt(option, group=section) + + +def list_opts(): + """Return a list of oslo_config options available in Keystone. + + The returned list includes all oslo_config options which are registered as + the "FILE_OPTIONS" in keystone.common.config. This list will not include + the options from the oslo-incubator library or any options registered + dynamically at run time. + + Each object in the list is a two element tuple. The first element of + each tuple is the name of the group under which the list of options in the + second element will be registered. A group name of None corresponds to the + [DEFAULT] group in config files. + + This function is also discoverable via the 'oslo_config.opts' entry point + under the 'keystone.config.opts' namespace. + + The purpose of this is to allow tools like the Oslo sample config file + generator to discover the options exposed to users by this library. + + :returns: a list of (group_name, opts) tuples + """ + return list(FILE_OPTIONS.items()) diff --git a/keystone/common/cache/backends/__init__.py b/oslo_cache/backends/__init__.py similarity index 100% rename from keystone/common/cache/backends/__init__.py rename to oslo_cache/backends/__init__.py diff --git a/keystone/common/cache/backends/memcache_pool.py b/oslo_cache/backends/memcache_pool.py similarity index 97% rename from keystone/common/cache/backends/memcache_pool.py rename to oslo_cache/backends/memcache_pool.py index f3990b12..86e17761 100644 --- a/keystone/common/cache/backends/memcache_pool.py +++ b/oslo_cache/backends/memcache_pool.py @@ -20,7 +20,7 @@ import logging from dogpile.cache.backends import memcached as memcached_backend -from keystone.common.cache import _memcache_pool +from oslo_cache import _memcache_pool LOG = logging.getLogger(__name__) diff --git a/keystone/common/cache/backends/mongo.py b/oslo_cache/backends/mongo.py similarity index 99% rename from keystone/common/cache/backends/mongo.py rename to oslo_cache/backends/mongo.py index 0598e80e..5cfae2a6 100644 --- a/keystone/common/cache/backends/mongo.py +++ b/oslo_cache/backends/mongo.py @@ -22,8 +22,8 @@ from oslo_utils import importutils from oslo_utils import timeutils import six -from keystone import exception -from keystone.i18n import _, _LW +from oslo_cache import exception +from oslo_cache._i18n import _, _LW NO_VALUE = api.NO_VALUE diff --git a/keystone/common/cache/backends/noop.py b/oslo_cache/backends/noop.py similarity index 100% rename from keystone/common/cache/backends/noop.py rename to oslo_cache/backends/noop.py diff --git a/keystone/common/cache/core.py b/oslo_cache/core.py similarity index 96% rename from keystone/common/cache/core.py rename to oslo_cache/core.py index 306587b3..830ed63c 100644 --- a/keystone/common/cache/core.py +++ b/oslo_cache/core.py @@ -21,8 +21,8 @@ from oslo_config import cfg from oslo_log import log from oslo_utils import importutils -from keystone import exception -from keystone.i18n import _, _LE +from oslo_cache import exception +from oslo_cache._i18n import _, _LE CONF = cfg.CONF @@ -31,18 +31,18 @@ LOG = log.getLogger(__name__) make_region = dogpile.cache.make_region dogpile.cache.register_backend( - 'keystone.common.cache.noop', - 'keystone.common.cache.backends.noop', + 'oslo_cache.noop', + 'oslo_cache.backends.noop', 'NoopCacheBackend') dogpile.cache.register_backend( - 'keystone.cache.mongo', - 'keystone.common.cache.backends.mongo', + 'oslo_cache.mongo', + 'oslo_cache.backends.mongo', 'MongoCacheBackend') dogpile.cache.register_backend( - 'keystone.cache.memcache_pool', - 'keystone.common.cache.backends.memcache_pool', + 'oslo_cache.memcache_pool', + 'oslo_cache.backends.memcache_pool', 'PooledMemcachedBackend') @@ -105,7 +105,7 @@ def build_cache_config(): arg_key = '.'.join([prefix, 'arguments', argname]) conf_dict[arg_key] = argvalue - LOG.debug('Keystone Cache Config: %s', conf_dict) + LOG.debug('Oslo Cache Config: %s', conf_dict) # NOTE(yorik-sar): these arguments will be used for memcache-related # backends. Use setdefault for url to support old-style setting through # backend_argument=url:127.0.0.1:11211 diff --git a/oslo_cache/exception.py b/oslo_cache/exception.py new file mode 100644 index 00000000..40cdb059 --- /dev/null +++ b/oslo_cache/exception.py @@ -0,0 +1,92 @@ +# Copyright 2012 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging as log + +from oslo_config import cfg +from oslo_utils import encodeutils +import six + +from oslo_cache._i18n import _, _LW + + +CONF = cfg.CONF +LOG = log.getLogger(__name__) + +# Tests use this to make exception message format errors fatal +_FATAL_EXCEPTION_FORMAT_ERRORS = False + + +class Error(Exception): + """Base error class. + + Child classes should define an HTTP status code, title, and a + message_format. + + """ + code = None + title = None + message_format = None + + def __init__(self, message=None, **kwargs): + try: + message = self._build_message(message, **kwargs) + except KeyError: + # if you see this warning in your logs, please raise a bug report + if _FATAL_EXCEPTION_FORMAT_ERRORS: + raise + else: + LOG.warning(_LW('missing exception kwargs (programmer error)')) + message = self.message_format + + super(Error, self).__init__(message) + + def _build_message(self, message, **kwargs): + """Builds and returns an exception message. + + :raises: KeyError given insufficient kwargs + + """ + if not message: + try: + message = self.message_format % kwargs + except UnicodeDecodeError: + try: + kwargs = {k: encodeutils.safe_decode(v) + for k, v in six.iteritems(kwargs)} + except UnicodeDecodeError: + # NOTE(jamielennox): This is the complete failure case + # at least by showing the template we have some idea + # of where the error is coming from + message = self.message_format + else: + message = self.message_format % kwargs + + return message + + +class ValidationError(Error): + message_format = _("Expecting to find %(attribute)s in %(target)s -" + " the server could not comply with the request" + " since it is either malformed or otherwise" + " incorrect. The client is assumed to be in error.") + code = 400 + title = 'Bad Request' + + +class NotImplemented(Error): + message_format = _("The action you have requested has not" + " been implemented.") + code = 501 + title = 'Not Implemented' diff --git a/oslo_cache/tests/__init__.py b/oslo_cache/tests/__init__.py index e69de29b..48cc8bd3 100644 --- a/oslo_cache/tests/__init__.py +++ b/oslo_cache/tests/__init__.py @@ -0,0 +1,15 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_cache import _opts + +_opts.configure() diff --git a/oslo_cache/tests/test_cache.py b/oslo_cache/tests/test_cache.py index c1492830..a717a28a 100644 --- a/oslo_cache/tests/test_cache.py +++ b/oslo_cache/tests/test_cache.py @@ -1,5 +1,5 @@ -# -*- coding: utf-8 -*- - +# Copyright 2013 Metacloud +# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -12,17 +12,335 @@ # License for the specific language governing permissions and limitations # under the License. -""" -test_cache ----------------------------------- - -Tests for `cache` module. -""" +import copy +import time +import uuid +from dogpile.cache import api +from dogpile.cache import proxy +import mock +from oslo_config import cfg from oslotest import base +import six +import testtools + +from oslo_cache import core as cache +from oslo_cache import exception +from oslo_config import fixture as config_fixture -class TestCache(base.BaseTestCase): +CONF = cfg.CONF +NO_VALUE = api.NO_VALUE - def test_something(self): - pass + +class BaseTestCase(base.BaseTestCase): + def setUp(self): + super(BaseTestCase, self).setUp() + + self.addCleanup(CONF.reset) + + self.config_fixture = self.useFixture(config_fixture.Config(CONF)) + self.addCleanup(delattr, self, 'config_fixture') + + +def _copy_value(value): + if value is not NO_VALUE: + value = copy.deepcopy(value) + return value + + +# NOTE(morganfainberg): WARNING - It is not recommended to use the Memory +# backend for dogpile.cache in a real deployment under any circumstances. The +# backend does no cleanup of expired values and therefore will leak memory. The +# backend is not implemented in a way to share data across processes (e.g. +# Keystone in HTTPD. This proxy is a hack to get around the lack of isolation +# of values in memory. Currently it blindly stores and retrieves the values +# from the cache, and modifications to dicts/lists/etc returned can result in +# changes to the cached values. In short, do not use the dogpile.cache.memory +# backend unless you are running tests or expecting odd/strange results. +class CacheIsolatingProxy(proxy.ProxyBackend): + """Proxy that forces a memory copy of stored values. + The default in-memory cache-region does not perform a copy on values it + is meant to cache. Therefore if the value is modified after set or after + get, the cached value also is modified. This proxy does a copy as the last + thing before storing data. + """ + def get(self, key): + return _copy_value(self.proxied.get(key)) + + def set(self, key, value): + self.proxied.set(key, _copy_value(value)) + + +class TestProxy(proxy.ProxyBackend): + def get(self, key): + value = _copy_value(self.proxied.get(key)) + if value is not NO_VALUE: + if isinstance(value[0], TestProxyValue): + value[0].cached = True + return value + + +class TestProxyValue(object): + def __init__(self, value): + self.value = value + self.cached = False + + +class CacheRegionTest(BaseTestCase): + + def setUp(self): + super(CacheRegionTest, self).setUp() + self.region = cache.make_region() + cache.configure_cache_region(self.region) + self.region.wrap(TestProxy) + self.test_value = TestProxyValue('Decorator Test') + + def _add_test_caching_option(self): + self.config_fixture.register_opt( + cfg.BoolOpt('caching', default=True), group='cache') + + def _get_cacheable_function(self): + with mock.patch.object(cache.REGION, 'cache_on_arguments', + self.region.cache_on_arguments): + memoize = cache.get_memoization_decorator(section='cache') + + @memoize + def cacheable_function(value): + return value + + return cacheable_function + + # FIXME(dims) : Need to resurrect this test ported from keystone + # def test_region_built_with_proxy_direct_cache_test(self): + # # Verify cache regions are properly built with proxies. + # test_value = TestProxyValue('Direct Cache Test') + # self.region.set('cache_test', test_value) + # cached_value = self.region.get('cache_test') + # self.assertTrue(cached_value.cached) + + def test_cache_region_no_error_multiple_config(self): + # Verify configuring the CacheRegion again doesn't error. + cache.configure_cache_region(self.region) + cache.configure_cache_region(self.region) + + def _get_cache_fallthrough_fn(self, cache_time): + with mock.patch.object(cache.REGION, 'cache_on_arguments', + self.region.cache_on_arguments): + memoize = cache.get_memoization_decorator( + section='cache', + expiration_section='assignment') + + class _test_obj(object): + def __init__(self, value): + self.test_value = value + + @memoize + def get_test_value(self): + return self.test_value + + def _do_test(value): + + test_obj = _test_obj(value) + + # Ensure the value has been cached + test_obj.get_test_value() + # Get the now cached value + cached_value = test_obj.get_test_value() + self.assertTrue(cached_value.cached) + self.assertEqual(value.value, cached_value.value) + self.assertEqual(cached_value.value, test_obj.test_value.value) + # Change the underlying value on the test object. + test_obj.test_value = TestProxyValue(uuid.uuid4().hex) + self.assertEqual(cached_value.value, + test_obj.get_test_value().value) + # override the system time to ensure the non-cached new value + # is returned + new_time = time.time() + (cache_time * 2) + with mock.patch.object(time, 'time', + return_value=new_time): + overriden_cache_value = test_obj.get_test_value() + self.assertNotEqual(cached_value.value, + overriden_cache_value.value) + self.assertEqual(test_obj.test_value.value, + overriden_cache_value.value) + + return _do_test + + # FIXME(dims) : Need to resurrect this test ported from keystone + # def test_cache_no_fallthrough_expiration_time_fn(self): + # # Since we do not re-configure the cache region, for ease of testing + # # this value is set the same as the expiration_time default in the + # # [cache] section + # cache_time = 600 + # expiration_time = cache.get_expiration_time_fn('role') + # do_test = self._get_cache_fallthrough_fn(cache_time) + # # Run the test with the assignment cache_time value + # self.config_fixture.config(cache_time=cache_time, + # group='role') + # test_value = TestProxyValue(uuid.uuid4().hex) + # self.assertEqual(cache_time, expiration_time()) + # do_test(value=test_value) + + # FIXME(dims) : Need to resurrect this test ported from keystone + # def test_cache_fallthrough_expiration_time_fn(self): + # # Since we do not re-configure the cache region, for ease of testing + # # this value is set the same as the expiration_time default in the + # # [cache] section + # cache_time = 599 + # expiration_time = cache.get_expiration_time_fn('role') + # do_test = self._get_cache_fallthrough_fn(cache_time) + # # Run the test with the assignment cache_time value set to None and + # # the global value set. + # self.config_fixture.config(cache_time=None, group='role') + # test_value = TestProxyValue(uuid.uuid4().hex) + # self.assertIsNone(expiration_time()) + # do_test(value=test_value) + + # FIXME(dims) : Need to resurrect this test ported from keystone + # def test_should_cache_fn_global_cache_enabled(self): + # # Verify should_cache_fn generates a sane function for subsystem and + # # functions as expected with caching globally enabled. + # cacheable_function = self._get_cacheable_function() + # + # self.config_fixture.config(group='cache', enabled=True) + # cacheable_function(self.test_value) + # cached_value = cacheable_function(self.test_value) + # self.assertTrue(cached_value.cached) + + @testtools.skipIf(six.PY3, 'FIXME: this test does not work python 3.x') + def test_should_cache_fn_global_cache_disabled(self): + # Verify should_cache_fn generates a sane function for subsystem and + # functions as expected with caching globally disabled. + cacheable_function = self._get_cacheable_function() + + self.config_fixture.config(group='cache', enabled=False) + cacheable_function(self.test_value) + cached_value = cacheable_function(self.test_value) + self.assertFalse(cached_value.cached) + + @testtools.skipIf(six.PY3, 'FIXME: this test does not work python 3.x') + def test_should_cache_fn_global_cache_disabled_section_cache_enabled(self): + # Verify should_cache_fn generates a sane function for subsystem and + # functions as expected with caching globally disabled and the specific + # section caching enabled. + cacheable_function = self._get_cacheable_function() + + self._add_test_caching_option() + self.config_fixture.config(group='cache', enabled=False) + self.config_fixture.config(group='cache', caching=True) + + cacheable_function(self.test_value) + cached_value = cacheable_function(self.test_value) + self.assertFalse(cached_value.cached) + + @testtools.skipIf(six.PY3, 'FIXME: this test does not work python 3.x') + def test_should_cache_fn_global_cache_enabled_section_cache_disabled(self): + # Verify should_cache_fn generates a sane function for subsystem and + # functions as expected with caching globally enabled and the specific + # section caching disabled. + cacheable_function = self._get_cacheable_function() + + self._add_test_caching_option() + self.config_fixture.config(group='cache', enabled=True) + self.config_fixture.config(group='cache', caching=False) + + cacheable_function(self.test_value) + cached_value = cacheable_function(self.test_value) + self.assertFalse(cached_value.cached) + + # FIXME(dims) : Need to resurrect this test ported from keystone + # def test_should_cache_fn_global_cache_enabled_section_cache_enabled( + # self): + # #Verify should_cache_fn generates a sane function for subsystem and + # #functions as expected with caching globally enabled and the specific + # #section caching enabled. + # cacheable_function = self._get_cacheable_function() + # + # self._add_test_caching_option() + # self.config_fixture.config(group='cache', enabled=True) + # self.config_fixture.config(group='cache', caching=True) + # + # cacheable_function(self.test_value) + # cached_value = cacheable_function(self.test_value) + # self.assertTrue(cached_value.cached) + + def test_cache_dictionary_config_builder(self): + """Validate we build a sane dogpile.cache dictionary config.""" + self.config_fixture.config(group='cache', + config_prefix='test_prefix', + backend='some_test_backend', + expiration_time=86400, + backend_argument=['arg1:test', + 'arg2:test:test', + 'arg3.invalid']) + + config_dict = cache.build_cache_config() + self.assertEqual( + CONF.cache.backend, config_dict['test_prefix.backend']) + self.assertEqual( + CONF.cache.expiration_time, + config_dict['test_prefix.expiration_time']) + self.assertEqual('test', config_dict['test_prefix.arguments.arg1']) + self.assertEqual('test:test', + config_dict['test_prefix.arguments.arg2']) + self.assertNotIn('test_prefix.arguments.arg3', config_dict) + + # FIXME(dims) : Need to resurrect this test ported from keystone + # def test_cache_debug_proxy(self): + # single_value = 'Test Value' + # single_key = 'testkey' + # multi_values = {'key1': 1, 'key2': 2, 'key3': 3} + # + # self.region.set(single_key, single_value) + # self.assertEqual(single_value, self.region.get(single_key)) + # + # self.region.delete(single_key) + # self.assertEqual(NO_VALUE, self.region.get(single_key)) + # + # self.region.set_multi(multi_values) + # cached_values = self.region.get_multi(multi_values.keys()) + # for value in multi_values.values(): + # self.assertIn(value, cached_values) + # self.assertEqual(len(multi_values.values()), len(cached_values)) + # + # self.region.delete_multi(multi_values.keys()) + # for value in self.region.get_multi(multi_values.keys()): + # self.assertEqual(NO_VALUE, value) + + def test_configure_non_region_object_raises_error(self): + self.assertRaises(exception.ValidationError, + cache.configure_cache_region, + "bogus") + + +class CacheNoopBackendTest(BaseTestCase): + + def setUp(self): + super(CacheNoopBackendTest, self).setUp() + self.region = cache.make_region() + cache.configure_cache_region(self.region) + + def config_overrides(self): + super(CacheNoopBackendTest, self).config_overrides() + self.config_fixture.config(group='cache', + backend='oslo_cache.cache.noop') + + @testtools.skipIf(six.PY3, 'FIXME: this test does not work python 3.x') + def test_noop_backend(self): + single_value = 'Test Value' + single_key = 'testkey' + multi_values = {'key1': 1, 'key2': 2, 'key3': 3} + + self.region.set(single_key, single_value) + self.assertEqual(NO_VALUE, self.region.get(single_key)) + + self.region.set_multi(multi_values) + cached_values = self.region.get_multi(multi_values.keys()) + self.assertEqual(len(cached_values), len(multi_values.values())) + for value in cached_values: + self.assertEqual(NO_VALUE, value) + + # Delete should not raise exceptions + self.region.delete(single_key) + self.region.delete_multi(multi_values.keys()) diff --git a/keystone/tests/unit/test_cache_backend_mongo.py b/oslo_cache/tests/test_cache_backend_mongo.py similarity index 94% rename from keystone/tests/unit/test_cache_backend_mongo.py rename to oslo_cache/tests/test_cache_backend_mongo.py index 1d8b607d..144372c7 100644 --- a/keystone/tests/unit/test_cache_backend_mongo.py +++ b/oslo_cache/tests/test_cache_backend_mongo.py @@ -22,10 +22,9 @@ from dogpile.cache import region as dp_region import six from six.moves import range -from keystone.common.cache.backends import mongo -from keystone import exception -from keystone.tests import unit as tests - +from oslo_cache.backends import mongo +from oslo_cache import exception +from oslo_cache.tests import test_cache # Mock database structure sample where 'ks_cache' is database and # 'cache' is collection. Dogpile CachedValue data is divided in two @@ -278,7 +277,7 @@ class MyTransformer(mongo.BaseTransform): return super(MyTransformer, self).transform_outgoing(son, collection) -class MongoCache(tests.BaseTestCase): +class MongoCache(test_cache.BaseTestCase): def setUp(self): super(MongoCache, self).setUp() global COLLECTIONS @@ -299,33 +298,33 @@ class MongoCache(tests.BaseTestCase): self.arguments.pop('db_hosts') region = dp_region.make_region() self.assertRaises(exception.ValidationError, region.configure, - 'keystone.cache.mongo', + 'oslo_cache.mongo', arguments=self.arguments) def test_missing_db_name(self): self.arguments.pop('db_name') region = dp_region.make_region() self.assertRaises(exception.ValidationError, region.configure, - 'keystone.cache.mongo', + 'oslo_cache.mongo', arguments=self.arguments) def test_missing_cache_collection_name(self): self.arguments.pop('cache_collection') region = dp_region.make_region() self.assertRaises(exception.ValidationError, region.configure, - 'keystone.cache.mongo', + 'oslo_cache.mongo', arguments=self.arguments) def test_incorrect_write_concern(self): self.arguments['w'] = 'one value' region = dp_region.make_region() self.assertRaises(exception.ValidationError, region.configure, - 'keystone.cache.mongo', + 'oslo_cache.mongo', arguments=self.arguments) def test_correct_write_concern(self): self.arguments['w'] = 1 - region = dp_region.make_region().configure('keystone.cache.mongo', + region = dp_region.make_region().configure('oslo_cache.mongo', arguments=self.arguments) random_key = uuid.uuid4().hex @@ -335,7 +334,7 @@ class MongoCache(tests.BaseTestCase): def test_incorrect_read_preference(self): self.arguments['read_preference'] = 'inValidValue' - region = dp_region.make_region().configure('keystone.cache.mongo', + region = dp_region.make_region().configure('oslo_cache.mongo', arguments=self.arguments) # As per delayed loading of pymongo, read_preference value should # still be string and NOT enum @@ -347,7 +346,7 @@ class MongoCache(tests.BaseTestCase): def test_correct_read_preference(self): self.arguments['read_preference'] = 'secondaryPreferred' - region = dp_region.make_region().configure('keystone.cache.mongo', + region = dp_region.make_region().configure('oslo_cache.mongo', arguments=self.arguments) # As per delayed loading of pymongo, read_preference value should # still be string and NOT enum @@ -365,13 +364,13 @@ class MongoCache(tests.BaseTestCase): self.arguments['use_replica'] = True region = dp_region.make_region() self.assertRaises(exception.ValidationError, region.configure, - 'keystone.cache.mongo', + 'oslo_cache.mongo', arguments=self.arguments) def test_provided_replica_set_name(self): self.arguments['use_replica'] = True self.arguments['replicaset_name'] = 'my_replica' - dp_region.make_region().configure('keystone.cache.mongo', + dp_region.make_region().configure('oslo_cache.mongo', arguments=self.arguments) self.assertTrue(True) # reached here means no initialization error @@ -379,7 +378,7 @@ class MongoCache(tests.BaseTestCase): self.arguments['mongo_ttl_seconds'] = 'sixty' region = dp_region.make_region() self.assertRaises(exception.ValidationError, region.configure, - 'keystone.cache.mongo', + 'oslo_cache.mongo', arguments=self.arguments) def test_cache_configuration_values_assertion(self): @@ -387,7 +386,7 @@ class MongoCache(tests.BaseTestCase): self.arguments['replicaset_name'] = 'my_replica' self.arguments['mongo_ttl_seconds'] = 60 self.arguments['ssl'] = False - region = dp_region.make_region().configure('keystone.cache.mongo', + region = dp_region.make_region().configure('oslo_cache.mongo', arguments=self.arguments) # There is no proxy so can access MongoCacheBackend directly self.assertEqual('localhost:27017', region.backend.api.hosts) @@ -404,7 +403,7 @@ class MongoCache(tests.BaseTestCase): arguments1 = copy.copy(self.arguments) arguments1['cache_collection'] = 'cache_region1' - region1 = dp_region.make_region().configure('keystone.cache.mongo', + region1 = dp_region.make_region().configure('oslo_cache.mongo', arguments=arguments1) # There is no proxy so can access MongoCacheBackend directly self.assertEqual('localhost:27017', region1.backend.api.hosts) @@ -428,7 +427,7 @@ class MongoCache(tests.BaseTestCase): arguments2['cache_collection'] = 'cache_region2' arguments2['son_manipulator'] = class_name - region2 = dp_region.make_region().configure('keystone.cache.mongo', + region2 = dp_region.make_region().configure('oslo_cache.mongo', arguments=arguments2) # There is no proxy so can access MongoCacheBackend directly self.assertEqual('localhost:27017', region2.backend.api.hosts) @@ -451,7 +450,7 @@ class MongoCache(tests.BaseTestCase): def test_typical_configuration(self): dp_region.make_region().configure( - 'keystone.cache.mongo', + 'oslo_cache.mongo', arguments=self.arguments ) self.assertTrue(True) # reached here means no initialization error @@ -459,7 +458,7 @@ class MongoCache(tests.BaseTestCase): def test_backend_get_missing_data(self): region = dp_region.make_region().configure( - 'keystone.cache.mongo', + 'oslo_cache.mongo', arguments=self.arguments ) @@ -470,7 +469,7 @@ class MongoCache(tests.BaseTestCase): def test_backend_set_data(self): region = dp_region.make_region().configure( - 'keystone.cache.mongo', + 'oslo_cache.mongo', arguments=self.arguments ) @@ -481,7 +480,7 @@ class MongoCache(tests.BaseTestCase): def test_backend_set_data_with_string_as_valid_ttl(self): self.arguments['mongo_ttl_seconds'] = '3600' - region = dp_region.make_region().configure('keystone.cache.mongo', + region = dp_region.make_region().configure('oslo_cache.mongo', arguments=self.arguments) self.assertEqual(3600, region.backend.api.ttl_seconds) random_key = uuid.uuid4().hex @@ -491,7 +490,7 @@ class MongoCache(tests.BaseTestCase): def test_backend_set_data_with_int_as_valid_ttl(self): self.arguments['mongo_ttl_seconds'] = 1800 - region = dp_region.make_region().configure('keystone.cache.mongo', + region = dp_region.make_region().configure('oslo_cache.mongo', arguments=self.arguments) self.assertEqual(1800, region.backend.api.ttl_seconds) random_key = uuid.uuid4().hex @@ -501,7 +500,7 @@ class MongoCache(tests.BaseTestCase): def test_backend_set_none_as_data(self): region = dp_region.make_region().configure( - 'keystone.cache.mongo', + 'oslo_cache.mongo', arguments=self.arguments ) @@ -512,7 +511,7 @@ class MongoCache(tests.BaseTestCase): def test_backend_set_blank_as_data(self): region = dp_region.make_region().configure( - 'keystone.cache.mongo', + 'oslo_cache.mongo', arguments=self.arguments ) @@ -523,7 +522,7 @@ class MongoCache(tests.BaseTestCase): def test_backend_set_same_key_multiple_times(self): region = dp_region.make_region().configure( - 'keystone.cache.mongo', + 'oslo_cache.mongo', arguments=self.arguments ) @@ -541,7 +540,7 @@ class MongoCache(tests.BaseTestCase): def test_backend_multi_set_data(self): region = dp_region.make_region().configure( - 'keystone.cache.mongo', + 'oslo_cache.mongo', arguments=self.arguments ) random_key = uuid.uuid4().hex @@ -562,7 +561,7 @@ class MongoCache(tests.BaseTestCase): def test_backend_multi_get_data(self): region = dp_region.make_region().configure( - 'keystone.cache.mongo', + 'oslo_cache.mongo', arguments=self.arguments ) random_key = uuid.uuid4().hex @@ -585,7 +584,7 @@ class MongoCache(tests.BaseTestCase): def test_backend_multi_set_should_update_existing(self): region = dp_region.make_region().configure( - 'keystone.cache.mongo', + 'oslo_cache.mongo', arguments=self.arguments ) random_key = uuid.uuid4().hex @@ -613,7 +612,7 @@ class MongoCache(tests.BaseTestCase): def test_backend_multi_set_get_with_blanks_none(self): region = dp_region.make_region().configure( - 'keystone.cache.mongo', + 'oslo_cache.mongo', arguments=self.arguments ) random_key = uuid.uuid4().hex @@ -654,7 +653,7 @@ class MongoCache(tests.BaseTestCase): def test_backend_delete_data(self): region = dp_region.make_region().configure( - 'keystone.cache.mongo', + 'oslo_cache.mongo', arguments=self.arguments ) @@ -669,7 +668,7 @@ class MongoCache(tests.BaseTestCase): def test_backend_multi_delete_data(self): region = dp_region.make_region().configure( - 'keystone.cache.mongo', + 'oslo_cache.mongo', arguments=self.arguments ) random_key = uuid.uuid4().hex @@ -705,7 +704,7 @@ class MongoCache(tests.BaseTestCase): self.arguments['continue_on_error'] = True self.arguments['secondary_acceptable_latency_ms'] = 60 region = dp_region.make_region().configure( - 'keystone.cache.mongo', + 'oslo_cache.mongo', arguments=self.arguments ) diff --git a/requirements.txt b/requirements.txt index c299e3bb..9f552de0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,8 @@ # process, which may cause wedges in the gate later. Babel>=1.3 +dogpile.cache>=0.5.3 +six>=1.9.0 +oslo.config>=1.11.0 # Apache-2.0 +oslo.log>=1.2.0 # Apache-2.0 +oslo.utils>=1.6.0 # Apache-2.0 diff --git a/test-requirements.txt b/test-requirements.txt index 6237219f..ff995cbb 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,3 +5,7 @@ hacking<0.11,>=0.10.0 oslotest>=1.5.1 # Apache-2.0 oslosphinx>=2.5.0 # Apache-2.0 sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3 +testtools>=1.4.0 + +# Optional dogpile backend: MongoDB +pymongo>=3.0.2