openstack/oslo.cache
Code Issues Proposed changes

Add socket keepalive options to oslo.cache

Browse Source 
This patch specifies a set of options required to setup the
socket keepalive of the dogpile.cache's pymemcache
backend [1][2]. This setup from those options can later on
be passed to this backend.

This patch also sets up the socket keepalive object based on
the configuration options passed via oslo.config and adds it
as an argument to be passed to the selected oslo.cache backend.

Dogpile.cache will be used as an interface between oslo.cache and
pymemcache [3].

[1] https://github.com/sqlalchemy/dogpile.cache/pull/205
[2]
b289c87bb8
[3]
https://dogpilecache.sqlalchemy.org/en/latest/api.html?highlight=keepalive#dogpile.cache.backends.memcached.PyMemcacheBackend.params.socket_keepalive

Change-Id: I501100e1a48cdd4e094c08046e2150405dcf371e
This commit is contained in:
Hervé Beraud 2021-08-06 11:49:17 +02:00
parent d2aef19023
commit f4fa6aa6fa
8 changed files with 144 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
doc/requirements.txt
View File

@ -11,6 +11,7 @@ sphinxcontrib-apidoc>=0.2.0 # BSD
# For autodoc builds
mock>=2.0.0 # BSD
oslotest>=3.2.0 # Apache-2.0
pymemcache>=3.5.0 # Apache-2.0
python-binary-memcached>=0.29.0 # MIT
python-memcached>=1.56 # PSF
etcd3gw>=0.2.0 # Apache-2.0

26
oslo_cache/_opts.py
View File

@ -146,6 +146,32 @@ FILE_OPTIONS = {
' the TLS context. It should be a string in the OpenSSL'
' cipher list format. If not specified, all OpenSSL enabled'
' ciphers will be available.'),
cfg.BoolOpt(
'enable_socket_keepalive',
default=False,
help="Global toggle for the socket keepalive of "
"dogpile's pymemcache backend"),
cfg.IntOpt(
'socket_keepalive_idle',
default=1,
min=0,
help='The time (in seconds) the connection needs to '
'remain idle before TCP starts sending keepalive probes. '
'Should be a positive integer most greater than zero.'),
cfg.IntOpt(
'socket_keepalive_interval',
default=1,
min=0,
help='The time (in seconds) between individual keepalive '
'probes. Should be a positive integer greater '
'than zero.'),
cfg.IntOpt(
'socket_keepalive_count',
default=1,
min=0,
help='The maximum number of keepalive probes TCP should '
'send before dropping the connection. Should be a '
'positive integer greater than zero.'),
],
}

23
oslo_cache/core.py
View File

@ -188,6 +188,29 @@ def _build_cache_config(conf):
conf_dict['%s.arguments.tls_context' % prefix] = tls_context
# NOTE(hberaud): Pymemcache support socket keepalive, If it is enable in
# our config then configure it to enable this feature.
# The socket keepalive feature means that pymemcache will be able to check
# your connected socket and determine whether the connection is still up
# and running or if it has broken.
# This could be used by users who want to handle fine grained failures.
if conf.cache.enable_socket_keepalive:
if conf.cache.backend != 'dogpile.cache.pymemcache':
msg = _(
"Socket keepalive is only supported by the "
"'dogpile.cache.pymemcache' backend."
)
raise exception.ConfigurationError(msg)
import pymemcache
socket_keepalive = pymemcache.KeepaliveOpts(
idle=conf.cache.socket_keepalive_idle,
intvl=conf.cache.socket_keepalive_interval,
cnt=conf.cache.socket_keepalive_count)
# As with the TLS context above, the config dict below will be
# consumed by dogpile.cache that will be used as a proxy between
# oslo.cache and pymemcache.
conf_dict['%s.arguments.socket_keepalive' % prefix] = socket_keepalive
return conf_dict

85
oslo_cache/tests/unit/test_cache_basics.py
View File

@ -21,6 +21,7 @@ from unittest import mock
from dogpile.cache import proxy
from oslo_config import cfg
from oslo_utils import uuidutils
from pymemcache import KeepaliveOpts
from oslo_cache import _opts
from oslo_cache import core as cache
@ -352,6 +353,90 @@ class CacheRegionTest(test_cache.BaseTestCase):
config_dict['test_prefix.arguments.tls_context'],
)
def test_cache_pymemcache_socket_kalive_enabled_with_wrong_backend(self):
"""Validate we build a config without the retry option when retry
is disabled.
"""
self.config_fixture.config(group='cache',
enabled=True,
config_prefix='test_prefix',
backend='oslo_cache.dict',
enable_socket_keepalive=True)
self.assertRaises(
exception.ConfigurationError,
cache._build_cache_config,
self.config_fixture.conf
)
def test_cache_pymemcache_socket_keepalive_disabled(self):
"""Validate we build a dogpile.cache dict config without keepalive."""
self.config_fixture.config(group='cache',
enabled=True,
config_prefix='test_prefix',
backend='dogpile.cache.pymemcache',
socket_keepalive_idle=2,
socket_keepalive_interval=2,
socket_keepalive_count=2)
config_dict = cache._build_cache_config(self.config_fixture.conf)
self.assertFalse(
self.config_fixture.conf.cache.enable_socket_keepalive)
self.assertNotIn(
'test_prefix.arguments.socket_keepalive', config_dict)
def test_cache_pymemcache_socket_keepalive_enabled(self):
"""Validate we build a dogpile.cache dict config with keepalive."""
self.config_fixture.config(group='cache',
enabled=True,
config_prefix='test_prefix',
backend='dogpile.cache.pymemcache',
enable_socket_keepalive=True)
config_dict = cache._build_cache_config(self.config_fixture.conf)
self.assertTrue(
self.config_fixture.conf.cache.enable_socket_keepalive)
self.assertIsInstance(
config_dict['test_prefix.arguments.socket_keepalive'],
KeepaliveOpts
)
def test_cache_pymemcache_socket_keepalive_with_config(self):
"""Validate we build a socket keepalive with the right config."""
self.config_fixture.config(group='cache',
enabled=True,
config_prefix='test_prefix',
backend='dogpile.cache.pymemcache',
enable_socket_keepalive=True,
socket_keepalive_idle=12,
socket_keepalive_interval=38,
socket_keepalive_count=42)
config_dict = cache._build_cache_config(self.config_fixture.conf)
self.assertTrue(
self.config_fixture.conf.cache.enable_socket_keepalive)
self.assertTrue(
config_dict['test_prefix.arguments.socket_keepalive'],
KeepaliveOpts
)
self.assertEqual(
12,
config_dict['test_prefix.arguments.socket_keepalive'].idle
)
self.assertEqual(
38,
config_dict['test_prefix.arguments.socket_keepalive'].intvl
)
self.assertEqual(
42,
config_dict['test_prefix.arguments.socket_keepalive'].cnt
)
def test_cache_dictionary_config_builder_flush_on_reconnect_enabled(self):
"""Validate we build a sane dogpile.cache dictionary config."""
self.config_fixture.config(group='cache',

6
releasenotes/notes/pymemcache_socket_keepalive-f91c69770961e2b6.yaml Normal file
View File

@ -0,0 +1,6 @@
---
features:
- |
New options (``enable_socket_keepalive``, ``socket_keepalive_idle``,
``socket_keepalive_interval``, ``socket_keepalive_count``) allow to use
and configure pymemcache's socket keepalive capabilities.

2
requirements.txt
View File

@ -2,7 +2,7 @@
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
dogpile.cache>=1.1.2 # BSD
dogpile.cache>=1.1.4 # BSD
oslo.config>=8.1.0 # Apache-2.0
oslo.i18n>=5.0.0 # Apache-2.0
oslo.log>=4.2.1 # Apache-2.0

1
setup.cfg
View File

@ -37,6 +37,7 @@ dogpile.cache =
[extras]
dogpile =
python-memcached>=1.56 # PSF
pymemcache>=3.5.0 # Apache-2.0
mongo =
pymongo!=3.1,>=3.0.2 # Apache-2.0
etcd3gw =

2
test-requirements.txt
View File

@ -8,7 +8,7 @@ pifpaf>=0.10.0 # Apache-2.0
bandit>=1.6.0,<1.7.0 # Apache-2.0
stestr>=2.0.0 # Apache-2.0
pre-commit>=2.6.0 # MIT
pymemcache>=3.4.0 # Apache-2.0
pymemcache>=3.5.0 # Apache-2.0
python-binary-memcached>=0.29.0 # MIT
python-memcached>=1.56 # PSF
pymongo!=3.1,>=3.0.2 # Apache-2.0