From f4fa6aa6fa2aca23a8f4f9e63c5a57dbcd2d1166 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Beraud?= Date: Fri, 6 Aug 2021 11:49:17 +0200 Subject: [PATCH] Add socket keepalive options to oslo.cache This patch specifies a set of options required to setup the socket keepalive of the dogpile.cache's pymemcache backend [1][2]. This setup from those options can later on be passed to this backend. This patch also sets up the socket keepalive object based on the configuration options passed via oslo.config and adds it as an argument to be passed to the selected oslo.cache backend. Dogpile.cache will be used as an interface between oslo.cache and pymemcache [3]. [1] https://github.com/sqlalchemy/dogpile.cache/pull/205 [2] https://github.com/pinterest/pymemcache/commit/b289c87bb89b3ab477bd5d92c8951ab42c923923 [3] https://dogpilecache.sqlalchemy.org/en/latest/api.html?highlight=keepalive#dogpile.cache.backends.memcached.PyMemcacheBackend.params.socket_keepalive Change-Id: I501100e1a48cdd4e094c08046e2150405dcf371e --- doc/requirements.txt | 1 + oslo_cache/_opts.py | 26 ++++++ oslo_cache/core.py | 23 +++++ oslo_cache/tests/unit/test_cache_basics.py | 85 +++++++++++++++++++ ...che_socket_keepalive-f91c69770961e2b6.yaml | 6 ++ requirements.txt | 2 +- setup.cfg | 1 + test-requirements.txt | 2 +- 8 files changed, 144 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/pymemcache_socket_keepalive-f91c69770961e2b6.yaml diff --git a/doc/requirements.txt b/doc/requirements.txt index c1c7c805..57674ca4 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -11,6 +11,7 @@ sphinxcontrib-apidoc>=0.2.0 # BSD # For autodoc builds mock>=2.0.0 # BSD oslotest>=3.2.0 # Apache-2.0 +pymemcache>=3.5.0 # Apache-2.0 python-binary-memcached>=0.29.0 # MIT python-memcached>=1.56 # PSF etcd3gw>=0.2.0 # Apache-2.0 diff --git a/oslo_cache/_opts.py b/oslo_cache/_opts.py index 167446b4..a008f1b7 100644 --- a/oslo_cache/_opts.py +++ b/oslo_cache/_opts.py @@ -146,6 +146,32 @@ FILE_OPTIONS = { ' the TLS context. It should be a string in the OpenSSL' ' cipher list format. If not specified, all OpenSSL enabled' ' ciphers will be available.'), + cfg.BoolOpt( + 'enable_socket_keepalive', + default=False, + help="Global toggle for the socket keepalive of " + "dogpile's pymemcache backend"), + cfg.IntOpt( + 'socket_keepalive_idle', + default=1, + min=0, + help='The time (in seconds) the connection needs to ' + 'remain idle before TCP starts sending keepalive probes. ' + 'Should be a positive integer most greater than zero.'), + cfg.IntOpt( + 'socket_keepalive_interval', + default=1, + min=0, + help='The time (in seconds) between individual keepalive ' + 'probes. Should be a positive integer greater ' + 'than zero.'), + cfg.IntOpt( + 'socket_keepalive_count', + default=1, + min=0, + help='The maximum number of keepalive probes TCP should ' + 'send before dropping the connection. Should be a ' + 'positive integer greater than zero.'), ], } diff --git a/oslo_cache/core.py b/oslo_cache/core.py index 6279166c..f0f32955 100644 --- a/oslo_cache/core.py +++ b/oslo_cache/core.py @@ -188,6 +188,29 @@ def _build_cache_config(conf): conf_dict['%s.arguments.tls_context' % prefix] = tls_context + # NOTE(hberaud): Pymemcache support socket keepalive, If it is enable in + # our config then configure it to enable this feature. + # The socket keepalive feature means that pymemcache will be able to check + # your connected socket and determine whether the connection is still up + # and running or if it has broken. + # This could be used by users who want to handle fine grained failures. + if conf.cache.enable_socket_keepalive: + if conf.cache.backend != 'dogpile.cache.pymemcache': + msg = _( + "Socket keepalive is only supported by the " + "'dogpile.cache.pymemcache' backend." + ) + raise exception.ConfigurationError(msg) + import pymemcache + socket_keepalive = pymemcache.KeepaliveOpts( + idle=conf.cache.socket_keepalive_idle, + intvl=conf.cache.socket_keepalive_interval, + cnt=conf.cache.socket_keepalive_count) + # As with the TLS context above, the config dict below will be + # consumed by dogpile.cache that will be used as a proxy between + # oslo.cache and pymemcache. + conf_dict['%s.arguments.socket_keepalive' % prefix] = socket_keepalive + return conf_dict diff --git a/oslo_cache/tests/unit/test_cache_basics.py b/oslo_cache/tests/unit/test_cache_basics.py index 88fadcb8..94ca32d1 100644 --- a/oslo_cache/tests/unit/test_cache_basics.py +++ b/oslo_cache/tests/unit/test_cache_basics.py @@ -21,6 +21,7 @@ from unittest import mock from dogpile.cache import proxy from oslo_config import cfg from oslo_utils import uuidutils +from pymemcache import KeepaliveOpts from oslo_cache import _opts from oslo_cache import core as cache @@ -352,6 +353,90 @@ class CacheRegionTest(test_cache.BaseTestCase): config_dict['test_prefix.arguments.tls_context'], ) + def test_cache_pymemcache_socket_kalive_enabled_with_wrong_backend(self): + """Validate we build a config without the retry option when retry + is disabled. + """ + self.config_fixture.config(group='cache', + enabled=True, + config_prefix='test_prefix', + backend='oslo_cache.dict', + enable_socket_keepalive=True) + + self.assertRaises( + exception.ConfigurationError, + cache._build_cache_config, + self.config_fixture.conf + ) + + def test_cache_pymemcache_socket_keepalive_disabled(self): + """Validate we build a dogpile.cache dict config without keepalive.""" + self.config_fixture.config(group='cache', + enabled=True, + config_prefix='test_prefix', + backend='dogpile.cache.pymemcache', + socket_keepalive_idle=2, + socket_keepalive_interval=2, + socket_keepalive_count=2) + + config_dict = cache._build_cache_config(self.config_fixture.conf) + + self.assertFalse( + self.config_fixture.conf.cache.enable_socket_keepalive) + self.assertNotIn( + 'test_prefix.arguments.socket_keepalive', config_dict) + + def test_cache_pymemcache_socket_keepalive_enabled(self): + """Validate we build a dogpile.cache dict config with keepalive.""" + self.config_fixture.config(group='cache', + enabled=True, + config_prefix='test_prefix', + backend='dogpile.cache.pymemcache', + enable_socket_keepalive=True) + + config_dict = cache._build_cache_config(self.config_fixture.conf) + + self.assertTrue( + self.config_fixture.conf.cache.enable_socket_keepalive) + + self.assertIsInstance( + config_dict['test_prefix.arguments.socket_keepalive'], + KeepaliveOpts + ) + + def test_cache_pymemcache_socket_keepalive_with_config(self): + """Validate we build a socket keepalive with the right config.""" + self.config_fixture.config(group='cache', + enabled=True, + config_prefix='test_prefix', + backend='dogpile.cache.pymemcache', + enable_socket_keepalive=True, + socket_keepalive_idle=12, + socket_keepalive_interval=38, + socket_keepalive_count=42) + + config_dict = cache._build_cache_config(self.config_fixture.conf) + + self.assertTrue( + self.config_fixture.conf.cache.enable_socket_keepalive) + + self.assertTrue( + config_dict['test_prefix.arguments.socket_keepalive'], + KeepaliveOpts + ) + self.assertEqual( + 12, + config_dict['test_prefix.arguments.socket_keepalive'].idle + ) + self.assertEqual( + 38, + config_dict['test_prefix.arguments.socket_keepalive'].intvl + ) + self.assertEqual( + 42, + config_dict['test_prefix.arguments.socket_keepalive'].cnt + ) + def test_cache_dictionary_config_builder_flush_on_reconnect_enabled(self): """Validate we build a sane dogpile.cache dictionary config.""" self.config_fixture.config(group='cache', diff --git a/releasenotes/notes/pymemcache_socket_keepalive-f91c69770961e2b6.yaml b/releasenotes/notes/pymemcache_socket_keepalive-f91c69770961e2b6.yaml new file mode 100644 index 00000000..c66136f6 --- /dev/null +++ b/releasenotes/notes/pymemcache_socket_keepalive-f91c69770961e2b6.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + New options (``enable_socket_keepalive``, ``socket_keepalive_idle``, + ``socket_keepalive_interval``, ``socket_keepalive_count``) allow to use + and configure pymemcache's socket keepalive capabilities. diff --git a/requirements.txt b/requirements.txt index 52eb1f23..4b7f67c2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -dogpile.cache>=1.1.2 # BSD +dogpile.cache>=1.1.4 # BSD oslo.config>=8.1.0 # Apache-2.0 oslo.i18n>=5.0.0 # Apache-2.0 oslo.log>=4.2.1 # Apache-2.0 diff --git a/setup.cfg b/setup.cfg index 2c4f807a..8249f496 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,6 +37,7 @@ dogpile.cache = [extras] dogpile = python-memcached>=1.56 # PSF + pymemcache>=3.5.0 # Apache-2.0 mongo = pymongo!=3.1,>=3.0.2 # Apache-2.0 etcd3gw = diff --git a/test-requirements.txt b/test-requirements.txt index 9188e991..bdcd518b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,7 +8,7 @@ pifpaf>=0.10.0 # Apache-2.0 bandit>=1.6.0,<1.7.0 # Apache-2.0 stestr>=2.0.0 # Apache-2.0 pre-commit>=2.6.0 # MIT -pymemcache>=3.4.0 # Apache-2.0 +pymemcache>=3.5.0 # Apache-2.0 python-binary-memcached>=0.29.0 # MIT python-memcached>=1.56 # PSF pymongo!=3.1,>=3.0.2 # Apache-2.0