Add native options for redis backend

This introduces a few new redis_* options to simplify the settings
required to use the redis backend. The main aim of this change is to
replace the requirement to inject url by backend_argument.

[cache]
backend=dogpile.cache.redis
redis_server=127.0.0.1:6379
redis_username=default
redis_password=a_big_secret
redis_socket_timeout=1.0

Closes-Bug: #2052351
Change-Id: Id72878f9cddaa99146eab5fb4ee76c8e6a633809

oslo_cache/_opts.py

@@ -128,6 +128,18 @@ FILE_OPTIONS = {
                    default='',
                    secret=True,
                    help='the password for the memcached which SASL enabled'),
+        cfg.StrOpt('redis_server',
+                   default='localhost:6379',
+                   help='Redis server in the format of "host:port"'),
+        cfg.StrOpt('redis_username',
+                   help='the user name for redis'),
+        cfg.StrOpt('redis_password',
+                   secret=True,
+                   help='the password for redis'),
+        cfg.FloatOpt('redis_socket_timeout',
+                     default=1.0,
+                     help='Timeout in seconds for every call to a server.'
+                     ' (dogpile.cache.redis backend only).'),
         cfg.BoolOpt('tls_enabled',
                     default=False,
                     help='Global toggle for TLS usage when communicating with'

oslo_cache/core.py

@@ -35,6 +35,7 @@ The library has special public value for nonexistent or expired keys called
     NO_VALUE = core.NO_VALUE
 """
 import ssl
+import urllib.parse
 
 import dogpile.cache
 from dogpile.cache import api
@@ -136,37 +137,63 @@ def _build_cache_config(conf):
         conf_dict[arg_key] = argvalue
 
         _LOG.debug('Oslo Cache Config: %s', conf_dict)
-    # NOTE(yorik-sar): these arguments will be used for memcache-related
+
-    # backends. Use setdefault for url to support old-style setting through
+    if conf.cache.backend == 'dogpile.cache.redis':
-    # backend_argument=url:127.0.0.1:11211
+        if conf.cache.redis_password is None:
-    #
+            netloc = conf.cache.redis_server
-    # NOTE(morgan): If requested by config, 'flush_on_reconnect' will be set
+        else:
-    # for pooled connections. This can ensure that stale data is never
+            if conf.cache.redis_username:
-    # consumed from a server that pops in/out due to a network partition
+                netloc = '%s:%s@%s' % (conf.cache.redis_username,
-    # or disconnect.
+                                       conf.cache.redis_password,
-    #
+                                       conf.cache.redis_server)
-    # See the help from python-memcached:
+            else:
-    #
+                netloc = ':%s@%s' % (conf.cache.redis_password,
-    # param flush_on_reconnect: optional flag which prevents a
+                                     conf.cache.redis_server)
-    #        scenario that can cause stale data to be read: If there's more
+
-    #        than one memcached server and the connection to one is
+        parts = urllib.parse.ParseResult(
-    #        interrupted, keys that mapped to that server will get
+            scheme=('rediss' if conf.cache.tls_enabled else 'redis'),
-    #        reassigned to another. If the first server comes back, those
+            netloc=netloc, path='', params='', query='', fragment='')
-    #        keys will map to it again. If it still has its data, get()s
+
-    #        can read stale data that was overwritten on another
+        conf_dict.setdefault(
-    #        server. This flag is off by default for backwards
+            '%s.arguments.url' % prefix,
-    #        compatibility.
+            urllib.parse.urlunparse(parts)
-    #
+        )
-    # The normal non-pooled clients connect explicitly on each use and
+        for arg in ('socket_timeout',):
-    # does not need the explicit flush_on_reconnect
+            value = getattr(conf.cache, 'redis_' + arg)
-    conf_dict.setdefault('%s.arguments.url' % prefix,
+            conf_dict['%s.arguments.%s' % (prefix, arg)] = value
-                         conf.cache.memcache_servers)
+    else:
-    for arg in ('dead_retry', 'socket_timeout', 'pool_maxsize',
+        # NOTE(yorik-sar): these arguments will be used for memcache-related
-                'pool_unused_timeout', 'pool_connection_get_timeout',
+        # backends. Use setdefault for url to support old-style setting through
-                'pool_flush_on_reconnect', 'sasl_enabled', 'username',
+        # backend_argument=url:127.0.0.1:11211
-                'password'):
+        #
-        value = getattr(conf.cache, 'memcache_' + arg)
+        # NOTE(morgan): If requested by config, 'flush_on_reconnect' will be
-        conf_dict['%s.arguments.%s' % (prefix, arg)] = value
+        # set for pooled connections. This can ensure that stale data is never
+        # consumed from a server that pops in/out due to a network partition
+        # or disconnect.
+        #
+        # See the help from python-memcached:
+        #
+        # param flush_on_reconnect: optional flag which prevents a
+        #        scenario that can cause stale data to be read: If there's more
+        #        than one memcached server and the connection to one is
+        #        interrupted, keys that mapped to that server will get
+        #        reassigned to another. If the first server comes back, those
+        #        keys will map to it again. If it still has its data, get()s
+        #        can read stale data that was overwritten on another
+        #        server. This flag is off by default for backwards
+        #        compatibility.
+        #
+        # The normal non-pooled clients connect explicitly on each use and
+        # does not need the explicit flush_on_reconnect
+        conf_dict.setdefault('%s.arguments.url' % prefix,
+                             conf.cache.memcache_servers)
+
+        for arg in ('dead_retry', 'socket_timeout', 'pool_maxsize',
+                    'pool_unused_timeout', 'pool_connection_get_timeout',
+                    'pool_flush_on_reconnect', 'sasl_enabled', 'username',
+                    'password'):
+            value = getattr(conf.cache, 'memcache_' + arg)
+            conf_dict['%s.arguments.%s' % (prefix, arg)] = value
 
     if conf.cache.tls_enabled:
         if conf.cache.backend in ('dogpile.cache.bmemcache',

oslo_cache/tests/unit/test_cache_basics.py

@@ -306,7 +306,9 @@ class CacheRegionTest(test_cache.BaseTestCase):
                                    tls_allowed_ciphers='allowed_ciphers')
 
         config_dict = cache._build_cache_config(self.config_fixture.conf)
-
+        self.assertEqual(
+            'redis://localhost:6379',
+            config_dict['test_prefix.arguments.url'])
         self.assertFalse(self.config_fixture.conf.cache.tls_enabled)
         self.assertNotIn('test_prefix.arguments.connection_kwargs',
                          config_dict)
@@ -351,6 +353,9 @@ class CacheRegionTest(test_cache.BaseTestCase):
         self.assertTrue(self.config_fixture.conf.cache.tls_enabled)
         self.assertIn('test_prefix.arguments.connection_kwargs',
                       config_dict)
+        self.assertEqual(
+            'rediss://localhost:6379',
+            config_dict['test_prefix.arguments.url'])
         self.assertEqual(
             {
                 'ssl_ca_certs': 'path_to_ca_file',
@@ -694,6 +699,36 @@ class CacheRegionTest(test_cache.BaseTestCase):
         self.assertFalse(config_dict['test_prefix.arguments'
                                      '.pool_flush_on_reconnect'])
 
+    def test_cache_dictionary_config_builder_redis(self):
+        """Validate the backend is reset to default if caching is disabled."""
+        self.config_fixture.config(group='cache',
+                                   config_prefix='test_prefix',
+                                   backend='dogpile.cache.redis',
+                                   redis_server='[::1]:6379',
+                                   redis_username='user',
+                                   redis_password='secrete')
+
+        config_dict = cache._build_cache_config(self.config_fixture.conf)
+        self.assertEqual(
+            'redis://user:secrete@[::1]:6379',
+            config_dict['test_prefix.arguments.url'])
+        self.assertEqual(
+            1.0, config_dict['test_prefix.arguments.socket_timeout'])
+
+    def test_cache_dictionary_config_builder_redis_with_auth(self):
+        """Validate the backend is reset to default if caching is disabled."""
+        self.config_fixture.config(group='cache',
+                                   config_prefix='test_prefix',
+                                   backend='dogpile.cache.redis',
+                                   redis_server='[::1]:6379',
+                                   redis_username='user',
+                                   redis_password='secrete')
+
+        config_dict = cache._build_cache_config(self.config_fixture.conf)
+        self.assertEqual(
+            'redis://user:secrete@[::1]:6379',
+            config_dict['test_prefix.arguments.url'])
+
     def test_cache_debug_proxy(self):
         single_value = 'Test Value'
         single_key = 'testkey'

releasenotes/notes/redis-backend-opts-27915f2b672512c9.yaml

@@ -0,0 +1,16 @@
+---
+features:
+  - |
+    The following new options are added. These options are used to customize
+    connections in the ``dogpile.cache.redis`` backend.
+
+    - ``redis_server``
+    - ``redis_username``
+    - ``redis_password``
+    - ``redis_socket_timeout``
+
+upgrade:
+  - |
+    The ``[cache] memcache_socket_timeout`` option no longer takes affect in
+    when the ``dogpile.cache.redis`` backend, which is the documented behavior.
+    Use the ``[cache] redis_socket_timeout`` option instead.