openstack/oslo.cache
Code Issues Proposed changes

Merge "Support expiration time in backend"

Browse Source
This commit is contained in:
Zuul 2024-10-30 14:47:14 +00:00 committed by Gerrit Code Review
commit dd347f1715
5 changed files with 97 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
oslo_cache/_opts.py
View File

@ -25,10 +25,17 @@ FILE_OPTIONS = {
'changed unless there is another dogpile.cache '
'region with the same configuration name.'),
cfg.IntOpt('expiration_time', default=600,
min=1,
help='Default TTL, in seconds, for any cached item in '
'the dogpile.cache region. This applies to any '
'cached method that doesn\'t have an explicit '
'cache expiration time defined for it.'),
cfg.IntOpt('backend_expiration_time',
min=1,
help='Expiration time in cache backend to purge '
'expired records automatically. This should be '
'greater than expiration_time and all cache_time '
'options'),
# NOTE(morganfainberg): It is recommended that either Redis or
# Memcached are used as the dogpile backend for real workloads. To
# prevent issues with the memory cache ending up in "production"

20
oslo_cache/core.py
View File

@ -214,6 +214,26 @@ def _build_cache_config(conf):
value = getattr(conf.cache, 'memcache_' + arg)
conf_dict[f'{prefix}.arguments.{arg}'] = value
if conf.cache.backend_expiration_time is not None:
if conf.cache.expiration_time > conf.cache.backend_expiration_time:
raise exception.ConfigurationError(
"backend_expiration_time should not be smaller than "
"expiration_time.")
if conf.cache.backend in ('dogpile.cache.pymemcache',
'dogpile.cache.memcached',
'dogpile.cache.pylibmc',
'oslo_cache.memcache_pool'):
conf_dict[f'{prefix}.arguments.memcached_expire_time'] = \
conf.cache.backend_expiration_time
elif conf.cache.backend in ('dogpile.cache.redis',
'dogpile.cache.redis_sentinel'):
conf_dict[f'{prefix}.arguments.redis_expiration_time'] = \
conf.cache.backend_expiration_time
else:
raise exception.ConfigurationError(
"Enabling backend expiration is not supported by"
"the %s driver", conf.cache.backend)
if conf.cache.tls_enabled:
if conf.cache.backend in ('dogpile.cache.bmemcache',
'dogpile.cache.pymemcache',

88
oslo_cache/tests/unit/test_cache_basics.py
View File

@ -242,7 +242,7 @@ class CacheRegionTest(test_cache.BaseTestCase):
cached_value = cacheable_function(self.test_value)
self.assertTrue(cached_value.cached)
def test_cache_dictionary_config_builder(self):
def test_cache_config_builder(self):
"""Validate we build a sane dogpile.cache dictionary config."""
self.config_fixture.config(group='cache',
config_prefix='test_prefix',
@ -264,7 +264,7 @@ class CacheRegionTest(test_cache.BaseTestCase):
config_dict['test_prefix.arguments.arg2'])
self.assertNotIn('test_prefix.arguments.arg3', config_dict)
def test_cache_dictionary_config_builder_global_disabled(self):
def test_cache_config_builder_global_disabled(self):
"""Validate the backend is reset to default if caching is disabled."""
self.config_fixture.config(group='cache',
enabled=False,
@ -277,7 +277,7 @@ class CacheRegionTest(test_cache.BaseTestCase):
_opts._DEFAULT_BACKEND,
config_dict['test_prefix.backend'])
def test_cache_dictionary_config_builder_tls_disabled(self):
def test_cache_config_builder_tls_disabled(self):
"""Validate the backend is reset to default if caching is disabled."""
self.config_fixture.config(group='cache',
enabled=True,
@ -295,7 +295,7 @@ class CacheRegionTest(test_cache.BaseTestCase):
ssl.create_default_context.assert_not_called()
self.assertNotIn('test_prefix.arguments.tls_context', config_dict)
def test_cache_dictionary_config_builder_tls_disabled_redis(self):
def test_cache_config_builder_tls_disabled_redis(self):
"""Validate the backend is reset to default if caching is disabled."""
self.config_fixture.config(group='cache',
enabled=True,
@ -314,7 +314,7 @@ class CacheRegionTest(test_cache.BaseTestCase):
self.assertNotIn('test_prefix.arguments.connection_kwargs',
config_dict)
def test_cache_dictionary_config_builder_tls_disabled_redis_sentinel(self):
def test_cache_config_builder_tls_disabled_redis_sentinel(self):
"""Validate the backend is reset to default if caching is disabled."""
self.config_fixture.config(group='cache',
enabled=True,
@ -332,7 +332,7 @@ class CacheRegionTest(test_cache.BaseTestCase):
self.assertNotIn('test_prefix.arguments.sentinel_kwargs',
config_dict)
def test_cache_dictionary_config_builder_tls_enabled(self):
def test_cache_config_builder_tls_enabled(self):
"""Validate the backend is reset to default if caching is disabled."""
self.config_fixture.config(group='cache',
enabled=True,
@ -356,7 +356,7 @@ class CacheRegionTest(test_cache.BaseTestCase):
config_dict['test_prefix.arguments.tls_context'],
)
def test_cache_dictionary_config_builder_tls_enabled_redis(self):
def test_cache_config_builder_tls_enabled_redis(self):
"""Validate the backend is reset to default if caching is disabled."""
self.config_fixture.config(group='cache',
enabled=True,
@ -384,7 +384,7 @@ class CacheRegionTest(test_cache.BaseTestCase):
config_dict['test_prefix.arguments.connection_kwargs'])
self.assertNotIn('test_prefix.arguments.sentinel_kwargs', config_dict)
def test_cache_dictionary_config_builder_tls_enabled_redis_sentinel(self):
def test_cache_config_builder_tls_enabled_redis_sentinel(self):
"""Validate the backend is reset to default if caching is disabled."""
self.config_fixture.config(group='cache',
enabled=True,
@ -420,7 +420,7 @@ class CacheRegionTest(test_cache.BaseTestCase):
config_dict['test_prefix.arguments.sentinel_kwargs'])
@mock.patch('oslo_cache.core._LOG')
def test_cache_dictionary_config_builder_fips_mode_supported(self, log):
def test_cache_config_builder_fips_mode_supported(self, log):
"""Validate the FIPS mode is supported."""
self.config_fixture.config(group='cache',
enabled=True,
@ -440,7 +440,7 @@ class CacheRegionTest(test_cache.BaseTestCase):
"Enforcing the use of the OpenSSL FIPS mode")
@mock.patch('oslo_cache.core._LOG')
def test_cache_dictionary_config_builder_fips_mode_unsupported(self, log):
def test_cache_config_builder_fips_mode_unsupported(self, log):
"""Validate the FIPS mode is not supported."""
self.config_fixture.config(group='cache',
enabled=True,
@ -458,7 +458,7 @@ class CacheRegionTest(test_cache.BaseTestCase):
cache._build_cache_config,
self.config_fixture.conf)
def test_cache_dictionary_config_builder_fips_mode_unsupported_redis(self):
def test_cache_config_builder_fips_mode_unsupported_redis(self):
"""Validate the FIPS mode is not supported."""
self.config_fixture.config(group='cache',
enabled=True,
@ -471,7 +471,7 @@ class CacheRegionTest(test_cache.BaseTestCase):
cache._build_cache_config,
self.config_fixture.conf)
def test_cache_dictionary_config_builder_tls_enabled_unsupported(self):
def test_cache_config_builder_tls_enabled_unsupported(self):
"""Validate the tls_enabled opiton is not supported.."""
self.config_fixture.config(group='cache',
enabled=True,
@ -485,7 +485,7 @@ class CacheRegionTest(test_cache.BaseTestCase):
self.config_fixture.conf)
ssl.create_default_context.assert_not_called()
def test_cache_dictionary_config_builder_tls_enabled_with_config(self):
def test_cache_config_builder_tls_enabled_with_config(self):
"""Validate the backend is reset to default if caching is disabled."""
self.config_fixture.config(group='cache',
enabled=True,
@ -724,7 +724,7 @@ class CacheRegionTest(test_cache.BaseTestCase):
100
)
def test_cache_dictionary_config_builder_flush_on_reconnect_enabled(self):
def test_cache_config_builder_flush_on_reconnect_enabled(self):
"""Validate we build a sane dogpile.cache dictionary config."""
self.config_fixture.config(group='cache',
enabled=True,
@ -739,7 +739,7 @@ class CacheRegionTest(test_cache.BaseTestCase):
self.assertTrue(config_dict['test_prefix.arguments'
'.pool_flush_on_reconnect'])
def test_cache_dictionary_config_builder_flush_on_reconnect_disabled(self):
def test_cache_config_builder_flush_on_reconnect_disabled(self):
"""Validate we build a sane dogpile.cache dictionary config."""
self.config_fixture.config(group='cache',
enabled=True,
@ -754,7 +754,7 @@ class CacheRegionTest(test_cache.BaseTestCase):
self.assertFalse(config_dict['test_prefix.arguments'
'.pool_flush_on_reconnect'])
def test_cache_dictionary_config_builder_redis(self):
def test_cache_config_builder_redis(self):
"""Validate we build a sane dogpile.cache dictionary config."""
self.config_fixture.config(group='cache',
config_prefix='test_prefix',
@ -768,7 +768,7 @@ class CacheRegionTest(test_cache.BaseTestCase):
self.assertEqual(
1.0, config_dict['test_prefix.arguments.socket_timeout'])
def test_cache_dictionary_config_builder_redis_with_db(self):
def test_cache_config_builder_redis_with_db(self):
"""Validate we build a sane dogpile.cache dictionary config."""
self.config_fixture.config(group='cache',
config_prefix='test_prefix',
@ -783,7 +783,7 @@ class CacheRegionTest(test_cache.BaseTestCase):
self.assertEqual(
1.0, config_dict['test_prefix.arguments.socket_timeout'])
def test_cache_dictionary_config_builder_redis_with_sock_to(self):
def test_cache_config_builder_redis_with_sock_to(self):
"""Validate we build a sane dogpile.cache dictionary config."""
self.config_fixture.config(group='cache',
config_prefix='test_prefix',
@ -798,7 +798,7 @@ class CacheRegionTest(test_cache.BaseTestCase):
self.assertEqual(
10.0, config_dict['test_prefix.arguments.socket_timeout'])
def test_cache_dictionary_config_builder_redis_with_keepalive(self):
def test_cache_config_builder_redis_with_keepalive(self):
"""Validate we build a sane dogpile.cache dictionary config."""
self.config_fixture.config(group='cache',
config_prefix='test_prefix',
@ -820,7 +820,7 @@ class CacheRegionTest(test_cache.BaseTestCase):
socket.TCP_KEEPCNT: 1,
}}, config_dict['test_prefix.arguments.connection_kwargs'])
def test_cache_dictionary_config_builder_redis_with_keepalive_params(self):
def test_cache_config_builder_redis_with_keepalive_params(self):
"""Validate we build a sane dogpile.cache dictionary config."""
self.config_fixture.config(group='cache',
config_prefix='test_prefix',
@ -845,7 +845,7 @@ class CacheRegionTest(test_cache.BaseTestCase):
socket.TCP_KEEPCNT: 4,
}}, config_dict['test_prefix.arguments.connection_kwargs'])
def test_cache_dictionary_config_builder_redis_with_auth(self):
def test_cache_config_builder_redis_with_auth(self):
"""Validate we build a sane dogpile.cache dictionary config."""
self.config_fixture.config(group='cache',
config_prefix='test_prefix',
@ -858,7 +858,7 @@ class CacheRegionTest(test_cache.BaseTestCase):
'redis://:secrete@[::1]:6379/0',
config_dict['test_prefix.arguments.url'])
def test_cache_dictionary_config_builder_redis_with_auth_and_user(self):
def test_cache_config_builder_redis_with_auth_and_user(self):
"""Validate we build a sane dogpile.cache dictionary config."""
self.config_fixture.config(group='cache',
config_prefix='test_prefix',
@ -872,7 +872,7 @@ class CacheRegionTest(test_cache.BaseTestCase):
'redis://user:secrete@[::1]:6379/0',
config_dict['test_prefix.arguments.url'])
def test_cache_dictionary_config_builder_redis_sentinel(self):
def test_cache_config_builder_redis_sentinel(self):
"""Validate we build a sane dogpile.cache dictionary config."""
self.config_fixture.config(group='cache',
enabled=True,
@ -896,7 +896,7 @@ class CacheRegionTest(test_cache.BaseTestCase):
self.assertNotIn('test_prefix.arguments.sentinel_kwargs',
config_dict)
def test_cache_dictionary_config_builder_redis_sentinel_with_db(self):
def test_cache_config_builder_redis_sentinel_with_db(self):
"""Validate we build a sane dogpile.cache dictionary config."""
self.config_fixture.config(group='cache',
enabled=True,
@ -921,7 +921,7 @@ class CacheRegionTest(test_cache.BaseTestCase):
self.assertNotIn('test_prefix.arguments.sentinel_kwargs',
config_dict)
def test_cache_dictionary_config_builder_redis_sentinel_with_sock_to(self):
def test_cache_config_builder_redis_sentinel_with_sock_to(self):
"""Validate we build a sane dogpile.cache dictionary config."""
self.config_fixture.config(group='cache',
enabled=True,
@ -946,7 +946,7 @@ class CacheRegionTest(test_cache.BaseTestCase):
self.assertNotIn('test_prefix.arguments.sentinel_kwargs',
config_dict)
def test_cache_dictionary_config_builder_redis_sentinel_with_auth(self):
def test_cache_config_builder_redis_sentinel_with_auth(self):
"""Validate we build a sane dogpile.cache dictionary config."""
self.config_fixture.config(group='cache',
enabled=True,
@ -979,6 +979,42 @@ class CacheRegionTest(test_cache.BaseTestCase):
self.assertEqual(
'secrete', config_dict['test_prefix.arguments.password'])
def test_cache_config_builder_with_backend_expiration(self):
"""Validate we build a sane dogpile.cache dictionary config."""
self.config_fixture.config(group='cache',
config_prefix='test_prefix',
backend='dogpile.cache.memcached',
backend_expiration_time=600)
config_dict = cache._build_cache_config(self.config_fixture.conf)
self.assertEqual(
600, config_dict['test_prefix.expiration_time'])
self.assertEqual(
600, config_dict['test_prefix.arguments.memcached_expire_time'])
def test_cache_config_builder_with_redis_backend_expiration(self):
"""Validate we build a sane dogpile.cache dictionary config."""
self.config_fixture.config(group='cache',
config_prefix='test_prefix',
backend='dogpile.cache.redis',
backend_expiration_time=600)
config_dict = cache._build_cache_config(self.config_fixture.conf)
self.assertEqual(
600, config_dict['test_prefix.expiration_time'])
self.assertEqual(
600, config_dict['test_prefix.arguments.redis_expiration_time'])
def test_cache_config_builder_with_backend_expiration_too_small(self):
"""Validate we build a sane dogpile.cache dictionary config."""
self.config_fixture.config(group='cache',
config_prefix='test_prefix',
backend='dogpile.cache.memcached',
backend_expiration_time=599)
self.assertRaises(exception.ConfigurationError,
cache._build_cache_config, self.config_fixture.conf)
def test_cache_debug_proxy(self):
single_value = 'Test Value'
single_key = 'testkey'

7
releasenotes/notes/enable_backend_expiration-252108d4f7bf3427.yaml Normal file
View File

@ -0,0 +1,7 @@
---
features:
- |
The new ``[cache] enable_backend_expiration`` option has been added. When
this option is set to ``True``, all cache records are added to the cache
backend in use with expiration time, so that expired records are
automatically purged by the reclaiming feature in the backend.

2
requirements.txt
View File

@ -1,4 +1,4 @@
dogpile.cache>=1.3.1 # BSD
dogpile.cache>=1.3.3 # BSD
oslo.config>=8.1.0 # Apache-2.0
oslo.i18n>=5.0.0 # Apache-2.0
oslo.log>=4.2.1 # Apache-2.0