redis: Add username

Redis introduced ACL feature in 4.0.0, and this feature is supported by
redis-py since 3.4.0[1]. When ACL is enabled, authentication requires
username in addition to password.

This also fixes how password is parsed from uri string. The parameter
description has saied that password should be passed in the following
format
 redis://[:<password>]<host>...
but the actual format current code expects is
 redis://[<password>]<host>...
which is not compliant with standard URL format.

[1] 8df8cd54d1

Change-Id: I55f268eea13c7b45dceae85cfac86f3fb1562f1a

releasenotes/notes/redis-username-98a265f61fca6a1c.yaml

@@ -0,0 +1,9 @@
+---
+features:
+  - |
+    Redis messaging store now supports authentication with username.
+
+deprecation:
+  - |
+    Password in redis uri will need to be prefixed by ':' in a future release.
+    Make sure all uri options are updated accordingly.

setup.cfg

@@ -96,7 +96,7 @@ oslo.policy.policies =
 mongodb =
     pymongo>=3.6.0 # Apache-2.0
 redis =
-    redis>=3.0.0 # MIT
+    redis>=3.4.0 # MIT
 mysql =
     PyMySQL>=0.8.0 # MIT License
 

test-requirements.txt

@@ -1,7 +1,7 @@
 hacking>=6.1.0,<6.2.0 # Apache-2.0
 
 # Backends
-redis>=3.0.0 # MIT
+redis>=3.4.0 # MIT
 pymongo>=3.6.0 # Apache-2.0
 python-swiftclient>=3.10.1 # Apache-2.0
 websocket-client>=0.44.0 # LGPLv2+

zaqar/conf/drivers_management_store_redis.py

@@ -39,7 +39,7 @@ uri = cfg.StrOpt(
           'string as "master=<name>". Finally, to connect '
           'to a local instance of Redis over a unix socket, '
           'you may use the form '
-          '"redis:[:password]@/path/to/redis.sock[?options]".'
+          '"redis://[:password]@/path/to/redis.sock[?options]".'
           ' In all forms, the "socket_timeout" option may be'
           'specified in the query string. Its value is '
           'given in seconds. If not provided, '

zaqar/conf/drivers_message_store_redis.py

@@ -40,7 +40,7 @@ uri = cfg.StrOpt(
           'string as "master=<name>". Finally, to connect '
           'to a local instance of Redis over a unix socket, '
           'you may use the form '
-          '"redis:[:password]@/path/to/redis.sock[?options]".'
+          '"redis://[:password]@/path/to/redis.sock[?options]".'
           ' In all forms, the "socket_timeout" option may be'
           'specified in the query string. Its value is '
           'given in seconds. If not provided, '

zaqar/storage/redis/driver.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from oslo_log import log as logging
 from osprofiler import profiler
 import redis
 import redis.sentinel
@@ -34,9 +35,11 @@ STRATEGY_TCP = 1
 STRATEGY_UNIX = 2
 STRATEGY_SENTINEL = 3
 
+LOG = logging.getLogger(__name__)
+
 
 class ConnectionURI(object):
-    def __init__(self, uri):  # noqa: C901
+    def __init__(self, uri):
         # TODO(prashanthr_): Add SSL support
         try:
             parsed_url = urllib.parse.urlparse(uri)
@@ -46,21 +49,23 @@ class ConnectionURI(object):
         if parsed_url.scheme != 'redis':
             raise errors.ConfigurationError(_('Invalid scheme in Redis URI'))
 
-        # NOTE(kgriffs): Python 2.6 has a bug that causes the
-        # query string to be appended to the path when given a
-        # hostless URL.
         path = parsed_url.path
-        if '?' in path:
+        query = parsed_url.query
-            path, sep, query = path.partition('?')
+        # NOTE(tkajinam): Replace '' by None
-        else:
+        self.password = parsed_url.password or None
-            query = parsed_url.query
+        self.username = parsed_url.username or None
-        # NOTE(gengchc2): Redis connection support password configure.
+
-        self.password = None
-        if '@' in path:
-            self.password, sep, path = path.partition('@')
         netloc = parsed_url.netloc
         if '@' in netloc:
-            self.password, sep, netloc = netloc.partition('@')
+            cred, sep, netloc = netloc.partition('@')
+
+        if self.username and not self.password:
+            # NOTE(tkajinam): This is kept for backword compatibility but
+            #                 should be removed after 2025.1
+            LOG.warning('Credential in redis uri does not contain \':\'. '
+                        'Make sure that \':\' is added before password.')
+            self.password = self.username
+            self.username = None
 
         query_params = dict(urllib.parse.parse_qsl(query))
 
@@ -315,6 +320,7 @@ def _get_redis_client(driver):
         sentinel = redis.sentinel.Sentinel(
             connection_uri.sentinels,
             db=connection_uri.dbid,
+            username=connection_uri.username,
             password=connection_uri.password,
             socket_timeout=connection_uri.socket_timeout)
 
@@ -328,11 +334,13 @@ def _get_redis_client(driver):
             host=connection_uri.hostname,
             port=connection_uri.port,
             db=connection_uri.dbid,
+            username=connection_uri.username,
             password=connection_uri.password,
             socket_timeout=connection_uri.socket_timeout)
     else:
         return redis.Redis(
             unix_socket_path=connection_uri.unix_socket_path,
             db=connection_uri.dbid,
+            username=connection_uri.username,
             password=connection_uri.password,
             socket_timeout=connection_uri.socket_timeout)

zaqar/tests/unit/storage/test_impl_redis.py

@@ -233,42 +233,99 @@ class RedisDriverTest(testing.TestBase):
         self.assertEqual(driver.STRATEGY_TCP, uri.strategy)
         self.assertEqual(6379, uri.port)
         self.assertEqual(0.1, uri.socket_timeout)
+        self.assertEqual(0, uri.dbid)
+        self.assertIsNone(uri.username)
+        self.assertIsNone(uri.password)
 
         uri = driver.ConnectionURI('redis://example.com:7777')
         self.assertEqual(driver.STRATEGY_TCP, uri.strategy)
         self.assertEqual(7777, uri.port)
+        self.assertEqual(0.1, uri.socket_timeout)
+        self.assertEqual(0, uri.dbid)
+        self.assertIsNone(uri.username)
+        self.assertIsNone(uri.password)
 
         uri = driver.ConnectionURI(
             'redis://example.com:7777?socket_timeout=1')
         self.assertEqual(driver.STRATEGY_TCP, uri.strategy)
         self.assertEqual(7777, uri.port)
         self.assertEqual(1.0, uri.socket_timeout)
+        self.assertEqual(0, uri.dbid)
+        self.assertIsNone(uri.username)
+        self.assertIsNone(uri.password)
 
         uri = driver.ConnectionURI(
-            'redis://test123@example.com:7777?socket_timeout=1&dbid=5')
+            'redis://:test123@example.com:7777?socket_timeout=1&dbid=5')
         self.assertEqual(driver.STRATEGY_TCP, uri.strategy)
         self.assertEqual(7777, uri.port)
         self.assertEqual(1.0, uri.socket_timeout)
         self.assertEqual(5, uri.dbid)
+        self.assertIsNone(uri.username)
+        self.assertEqual('test123', uri.password)
+
+        # NOTE(tkajinam): Test fallback for backword compatibility
+        uri = driver.ConnectionURI('redis://test123@example.com')
+        self.assertEqual(driver.STRATEGY_TCP, uri.strategy)
+        self.assertEqual(6379, uri.port)
+        self.assertEqual(0.1, uri.socket_timeout)
+        self.assertEqual(0, uri.dbid)
+        self.assertIsNone(uri.username)
+        self.assertEqual('test123', uri.password)
+
+        uri = driver.ConnectionURI(
+            'redis://default:test123@example.com')
+        self.assertEqual(driver.STRATEGY_TCP, uri.strategy)
+        self.assertEqual(6379, uri.port)
+        self.assertEqual(0.1, uri.socket_timeout)
+        self.assertEqual(0, uri.dbid)
+        self.assertEqual('default', uri.username)
         self.assertEqual('test123', uri.password)
 
     def test_connection_uri_unix_socket(self):
-        uri = driver.ConnectionURI('redis:/tmp/redis.sock')
+        uri = driver.ConnectionURI('redis:///tmp/redis.sock')
         self.assertEqual(driver.STRATEGY_UNIX, uri.strategy)
         self.assertEqual('/tmp/redis.sock', uri.unix_socket_path)
         self.assertEqual(0.1, uri.socket_timeout)
+        self.assertEqual(0, uri.dbid)
+        self.assertIsNone(uri.username)
+        self.assertIsNone(uri.password)
 
-        uri = driver.ConnectionURI('redis:/tmp/redis.sock?socket_timeout=1.5')
+        uri = driver.ConnectionURI(
+            'redis:///tmp/redis.sock?socket_timeout=1.5')
         self.assertEqual(driver.STRATEGY_UNIX, uri.strategy)
         self.assertEqual('/tmp/redis.sock', uri.unix_socket_path)
         self.assertEqual(1.5, uri.socket_timeout)
+        self.assertEqual(0, uri.dbid)
+        self.assertIsNone(uri.username)
+        self.assertIsNone(uri.password)
 
         uri = driver.ConnectionURI(
-            'redis:test123@/tmp/redis.sock?socket_timeout=1.5&dbid=5')
+            'redis://:test123@/tmp/redis.sock?'
+            'socket_timeout=1.5&dbid=5')
         self.assertEqual(driver.STRATEGY_UNIX, uri.strategy)
         self.assertEqual('/tmp/redis.sock', uri.unix_socket_path)
         self.assertEqual(1.5, uri.socket_timeout)
         self.assertEqual(5, uri.dbid)
+        self.assertIsNone(uri.username)
+        self.assertEqual('test123', uri.password)
+
+        # NOTE(tkajinam): Test fallback for backword compatibility
+        uri = driver.ConnectionURI(
+            'redis://test123@/tmp/redis.sock')
+        self.assertEqual(driver.STRATEGY_UNIX, uri.strategy)
+        self.assertEqual('/tmp/redis.sock', uri.unix_socket_path)
+        self.assertEqual(0.1, uri.socket_timeout)
+        self.assertEqual(0, uri.dbid)
+        self.assertIsNone(uri.username)
+        self.assertEqual('test123', uri.password)
+
+        uri = driver.ConnectionURI(
+            'redis://default:test123@/tmp/redis.sock')
+        self.assertEqual(driver.STRATEGY_UNIX, uri.strategy)
+        self.assertEqual('/tmp/redis.sock', uri.unix_socket_path)
+        self.assertEqual(0.1, uri.socket_timeout)
+        self.assertEqual(0, uri.dbid)
+        self.assertEqual('default', uri.username)
         self.assertEqual('test123', uri.password)
 
     def test_connection_uri_sentinel(self):
@@ -277,18 +334,27 @@ class RedisDriverTest(testing.TestBase):
         self.assertEqual([('s1', 26379)], uri.sentinels)
         self.assertEqual('dumbledore', uri.master)
         self.assertEqual(0.1, uri.socket_timeout)
+        self.assertEqual(0, uri.dbid)
+        self.assertIsNone(uri.username)
+        self.assertIsNone(uri.password)
 
         uri = driver.ConnectionURI('redis://s1,s2?master=dumbledore')
         self.assertEqual(driver.STRATEGY_SENTINEL, uri.strategy)
         self.assertEqual([('s1', 26379), ('s2', 26379)], uri.sentinels)
         self.assertEqual('dumbledore', uri.master)
         self.assertEqual(0.1, uri.socket_timeout)
+        self.assertEqual(0, uri.dbid)
+        self.assertIsNone(uri.username)
+        self.assertIsNone(uri.password)
 
         uri = driver.ConnectionURI('redis://s1:26389,s1?master=dumbledore')
         self.assertEqual(driver.STRATEGY_SENTINEL, uri.strategy)
         self.assertEqual([('s1', 26389), ('s1', 26379)], uri.sentinels)
         self.assertEqual('dumbledore', uri.master)
         self.assertEqual(0.1, uri.socket_timeout)
+        self.assertEqual(0, uri.dbid)
+        self.assertIsNone(uri.username)
+        self.assertIsNone(uri.password)
 
         uri = driver.ConnectionURI(
             'redis://[::1]:26389,[::2]?master=dumbledore')
@@ -296,6 +362,9 @@ class RedisDriverTest(testing.TestBase):
         self.assertEqual([('::1', 26389), ('::2', 26379)], uri.sentinels)
         self.assertEqual('dumbledore', uri.master)
         self.assertEqual(0.1, uri.socket_timeout)
+        self.assertEqual(0, uri.dbid)
+        self.assertIsNone(uri.username)
+        self.assertIsNone(uri.password)
 
         uri = driver.ConnectionURI(
             'redis://s1?master=dumbledore&socket_timeout=0.5')
@@ -303,14 +372,39 @@ class RedisDriverTest(testing.TestBase):
         self.assertEqual([('s1', 26379)], uri.sentinels)
         self.assertEqual('dumbledore', uri.master)
         self.assertEqual(0.5, uri.socket_timeout)
+        self.assertEqual(0, uri.dbid)
+        self.assertIsNone(uri.username)
+        self.assertIsNone(uri.password)
 
         uri = driver.ConnectionURI(
-            'redis://test123@s1?master=dumbledore&socket_timeout=0.5&dbid=5')
+            'redis://:test123@s1?master=dumbledore&socket_timeout=0.5&dbid=5')
         self.assertEqual(driver.STRATEGY_SENTINEL, uri.strategy)
         self.assertEqual([('s1', 26379)], uri.sentinels)
         self.assertEqual('dumbledore', uri.master)
         self.assertEqual(0.5, uri.socket_timeout)
         self.assertEqual(5, uri.dbid)
+        self.assertIsNone(uri.username)
+        self.assertEqual('test123', uri.password)
+
+        # NOTE(tkajinam): Test fallback for backword compatibility
+        uri = driver.ConnectionURI(
+            'redis://test123@s1?master=dumbledore')
+        self.assertEqual(driver.STRATEGY_SENTINEL, uri.strategy)
+        self.assertEqual([('s1', 26379)], uri.sentinels)
+        self.assertEqual('dumbledore', uri.master)
+        self.assertEqual(0.1, uri.socket_timeout)
+        self.assertEqual(0, uri.dbid)
+        self.assertIsNone(uri.username)
+        self.assertEqual('test123', uri.password)
+
+        uri = driver.ConnectionURI(
+            'redis://default:test123@s1?master=dumbledore')
+        self.assertEqual(driver.STRATEGY_SENTINEL, uri.strategy)
+        self.assertEqual([('s1', 26379)], uri.sentinels)
+        self.assertEqual('dumbledore', uri.master)
+        self.assertEqual(0.1, uri.socket_timeout)
+        self.assertEqual(0, uri.dbid)
+        self.assertEqual('default', uri.username)
         self.assertEqual('test123', uri.password)