openstack/oslo.cache
Code Issues Proposed changes

Add native options for redis backend

Browse Source 
This introduces a few new redis_* options to simplify the settings
required to use the redis backend. The main aim of this change is to
replace the requirement to inject url by backend_argument.

[cache]
backend=dogpile.cache.redis
redis_server=127.0.0.1:6379
redis_username=default
redis_password=a_big_secret
redis_socket_timeout=1.0

Closes-Bug: #2052351
Change-Id: Id72878f9cddaa99146eab5fb4ee76c8e6a633809
This commit is contained in:
Takashi Kajinami 2024-02-03 21:10:28 +09:00
parent 31201ce71c
commit 28411250da
4 changed files with 122 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
oslo_cache/_opts.py
View File

@ -128,6 +128,18 @@ FILE_OPTIONS = {
default='',
secret=True,
help='the password for the memcached which SASL enabled'),
cfg.StrOpt('redis_server',
default='localhost:6379',
help='Redis server in the format of "host:port"'),
cfg.StrOpt('redis_username',
help='the user name for redis'),
cfg.StrOpt('redis_password',
secret=True,
help='the password for redis'),
cfg.FloatOpt('redis_socket_timeout',
default=1.0,
help='Timeout in seconds for every call to a server.'
' (dogpile.cache.redis backend only).'),
cfg.BoolOpt('tls_enabled',
default=False,
help='Global toggle for TLS usage when communicating with'

89
oslo_cache/core.py
View File

@ -35,6 +35,7 @@ The library has special public value for nonexistent or expired keys called
NO_VALUE = core.NO_VALUE
"""
import ssl
import urllib.parse
import dogpile.cache
from dogpile.cache import api
@ -136,37 +137,63 @@ def _build_cache_config(conf):
conf_dict[arg_key] = argvalue
_LOG.debug('Oslo Cache Config: %s', conf_dict)
# NOTE(yorik-sar): these arguments will be used for memcache-related
# backends. Use setdefault for url to support old-style setting through
# backend_argument=url:127.0.0.1:11211
#
# NOTE(morgan): If requested by config, 'flush_on_reconnect' will be set
# for pooled connections. This can ensure that stale data is never
# consumed from a server that pops in/out due to a network partition
# or disconnect.
#
# See the help from python-memcached:
#
# param flush_on_reconnect: optional flag which prevents a
# scenario that can cause stale data to be read: If there's more
# than one memcached server and the connection to one is
# interrupted, keys that mapped to that server will get
# reassigned to another. If the first server comes back, those
# keys will map to it again. If it still has its data, get()s
# can read stale data that was overwritten on another
# server. This flag is off by default for backwards
# compatibility.
#
# The normal non-pooled clients connect explicitly on each use and
# does not need the explicit flush_on_reconnect
conf_dict.setdefault('%s.arguments.url' % prefix,
conf.cache.memcache_servers)
for arg in ('dead_retry', 'socket_timeout', 'pool_maxsize',
'pool_unused_timeout', 'pool_connection_get_timeout',
'pool_flush_on_reconnect', 'sasl_enabled', 'username',
'password'):
value = getattr(conf.cache, 'memcache_' + arg)
conf_dict['%s.arguments.%s' % (prefix, arg)] = value
if conf.cache.backend == 'dogpile.cache.redis':
if conf.cache.redis_password is None:
netloc = conf.cache.redis_server
else:
if conf.cache.redis_username:
netloc = '%s:%s@%s' % (conf.cache.redis_username,
conf.cache.redis_password,
conf.cache.redis_server)
else:
netloc = ':%s@%s' % (conf.cache.redis_password,
conf.cache.redis_server)
parts = urllib.parse.ParseResult(
scheme=('rediss' if conf.cache.tls_enabled else 'redis'),
netloc=netloc, path='', params='', query='', fragment='')
conf_dict.setdefault(
'%s.arguments.url' % prefix,
urllib.parse.urlunparse(parts)
)
for arg in ('socket_timeout',):
value = getattr(conf.cache, 'redis_' + arg)
conf_dict['%s.arguments.%s' % (prefix, arg)] = value
else:
# NOTE(yorik-sar): these arguments will be used for memcache-related
# backends. Use setdefault for url to support old-style setting through
# backend_argument=url:127.0.0.1:11211
#
# NOTE(morgan): If requested by config, 'flush_on_reconnect' will be
# set for pooled connections. This can ensure that stale data is never
# consumed from a server that pops in/out due to a network partition
# or disconnect.
#
# See the help from python-memcached:
#
# param flush_on_reconnect: optional flag which prevents a
# scenario that can cause stale data to be read: If there's more
# than one memcached server and the connection to one is
# interrupted, keys that mapped to that server will get
# reassigned to another. If the first server comes back, those
# keys will map to it again. If it still has its data, get()s
# can read stale data that was overwritten on another
# server. This flag is off by default for backwards
# compatibility.
#
# The normal non-pooled clients connect explicitly on each use and
# does not need the explicit flush_on_reconnect
conf_dict.setdefault('%s.arguments.url' % prefix,
conf.cache.memcache_servers)
for arg in ('dead_retry', 'socket_timeout', 'pool_maxsize',
'pool_unused_timeout', 'pool_connection_get_timeout',
'pool_flush_on_reconnect', 'sasl_enabled', 'username',
'password'):
value = getattr(conf.cache, 'memcache_' + arg)
conf_dict['%s.arguments.%s' % (prefix, arg)] = value
if conf.cache.tls_enabled:
if conf.cache.backend in ('dogpile.cache.bmemcache',

37
oslo_cache/tests/unit/test_cache_basics.py
View File

@ -306,7 +306,9 @@ class CacheRegionTest(test_cache.BaseTestCase):
tls_allowed_ciphers='allowed_ciphers')
config_dict = cache._build_cache_config(self.config_fixture.conf)
self.assertEqual(
'redis://localhost:6379',
config_dict['test_prefix.arguments.url'])
self.assertFalse(self.config_fixture.conf.cache.tls_enabled)
self.assertNotIn('test_prefix.arguments.connection_kwargs',
config_dict)
@ -351,6 +353,9 @@ class CacheRegionTest(test_cache.BaseTestCase):
self.assertTrue(self.config_fixture.conf.cache.tls_enabled)
self.assertIn('test_prefix.arguments.connection_kwargs',
config_dict)
self.assertEqual(
'rediss://localhost:6379',
config_dict['test_prefix.arguments.url'])
self.assertEqual(
{
'ssl_ca_certs': 'path_to_ca_file',
@ -694,6 +699,36 @@ class CacheRegionTest(test_cache.BaseTestCase):
self.assertFalse(config_dict['test_prefix.arguments'
'.pool_flush_on_reconnect'])
def test_cache_dictionary_config_builder_redis(self):
"""Validate the backend is reset to default if caching is disabled."""
self.config_fixture.config(group='cache',
config_prefix='test_prefix',
backend='dogpile.cache.redis',
redis_server='[::1]:6379',
redis_username='user',
redis_password='secrete')
config_dict = cache._build_cache_config(self.config_fixture.conf)
self.assertEqual(
'redis://user:secrete@[::1]:6379',
config_dict['test_prefix.arguments.url'])
self.assertEqual(
1.0, config_dict['test_prefix.arguments.socket_timeout'])
def test_cache_dictionary_config_builder_redis_with_auth(self):
"""Validate the backend is reset to default if caching is disabled."""
self.config_fixture.config(group='cache',
config_prefix='test_prefix',
backend='dogpile.cache.redis',
redis_server='[::1]:6379',
redis_username='user',
redis_password='secrete')
config_dict = cache._build_cache_config(self.config_fixture.conf)
self.assertEqual(
'redis://user:secrete@[::1]:6379',
config_dict['test_prefix.arguments.url'])
def test_cache_debug_proxy(self):
single_value = 'Test Value'
single_key = 'testkey'

16
releasenotes/notes/redis-backend-opts-27915f2b672512c9.yaml Normal file
View File

@ -0,0 +1,16 @@
---
features:
- |
The following new options are added. These options are used to customize
connections in the ``dogpile.cache.redis`` backend.
- ``redis_server``
- ``redis_username``
- ``redis_password``
- ``redis_socket_timeout``
upgrade:
- |
The ``[cache] memcache_socket_timeout`` option no longer takes affect in
when the ``dogpile.cache.redis`` backend, which is the documented behavior.
Use the ``[cache] redis_socket_timeout`` option instead.