From 788d3c4969e3446778496f3a9055f654602ae2c1 Mon Sep 17 00:00:00 2001 From: Michal Arbet Date: Wed, 10 Feb 2021 14:47:17 +0100 Subject: [PATCH] Switch to eventlet-safe oslo.cache's MemcacheClientPool In past days there were discussions about various issues with memcached connections [1][2][3]. After investigation it looks like common root cause for above problems is keystonemiddleware. More precisely said the way how keystonemiddleware is caching tokens. Currently it's using some home-made CachePool with direct usage of memcached library, moreover it looks like its approach is not eventlet-safe. Discussion can be mainly found in [4]. Fortunately keystonemiddleware can use "advanced cache pool", which is oslo.cache's implementation and was added long time ago [5], but it is turned on only if memcache_use_advanced_pool=True. This patch is switching to more elaborated oslo.cache CachePool and adding deprecation warning about eventlet-unsafe variant of keystonemiddleware's memcache pool. How to reproduce ? with memcache_use_advanced_pool=False 1. Build clean ENV of openstack 2. Deploy core projects (keystone,glance,nova,placement...) 3. Run while true; do COMMAND FOR SERVICE; done - several bashes, in parallel (5-7) COMMAND FOR SERVICE: - openstack network list - openstack volume list - openstack server list - openstack image list 4. Check memcached connections (which will grow up): - ss | grep 11211 | wc -l every second How to fix and test it ? Repeat above, to fix: - with memcache_use_advanced_pool=True OR - apply this patch Compare measurements in graph. [1] https://bugs.launchpad.net/keystonemiddleware/+bug/1892852 [2] https://bugs.launchpad.net/oslo.cache/+bug/1888394 [3] https://bugs.launchpad.net/keystonemiddleware/+bug/1883659 [4] https://review.opendev.org/c/openstack/oslo.cache/+/742193 [5] https://review.opendev.org/c/openstack/keystonemiddleware/+/268664 Closes-Bug: #1883659 Closes-Bug: #1892852 Closes-Bug: #1888394 Change-Id: I0e96334b65a0bf369ebf1d88651d13feb8d2ecac --- keystonemiddleware/auth_token/_cache.py | 9 +++++++- keystonemiddleware/auth_token/_opts.py | 5 ++--- ...e-memcacheclientpool-f8b4a6733513d73e.yaml | 22 +++++++++++++++++++ 3 files changed, 32 insertions(+), 4 deletions(-) create mode 100644 releasenotes/notes/deprecate-eventlet-unsafe-memcacheclientpool-f8b4a6733513d73e.yaml diff --git a/keystonemiddleware/auth_token/_cache.py b/keystonemiddleware/auth_token/_cache.py index 8951b375..94aa7cde 100644 --- a/keystonemiddleware/auth_token/_cache.py +++ b/keystonemiddleware/auth_token/_cache.py @@ -124,7 +124,7 @@ class TokenCache(object): def __init__(self, log, cache_time=None, env_cache_name=None, memcached_servers=None, - use_advanced_pool=False, dead_retry=None, socket_timeout=None, + use_advanced_pool=True, dead_retry=None, socket_timeout=None, **kwargs): self._LOG = log self._cache_time = cache_time @@ -150,6 +150,13 @@ class TokenCache(object): **self._memcache_pool_options) else: + if not self._use_advanced_pool: + self._LOG.warning( + "Using the eventlet-unsafe cache pool is deprecated." + "It is recommended to use eventlet-safe cache pool" + "implementation from oslo.cache. This can be enabled" + "through config option memcache_use_advanced_pool = True") + return _CachePool(self._memcached_servers, self._LOG) def initialize(self, env): diff --git a/keystonemiddleware/auth_token/_opts.py b/keystonemiddleware/auth_token/_opts.py index 15dd4f64..1ddef7d3 100644 --- a/keystonemiddleware/auth_token/_opts.py +++ b/keystonemiddleware/auth_token/_opts.py @@ -145,10 +145,9 @@ _OPTS = [ help='(Optional) Number of seconds that an operation will wait ' 'to get a memcached client connection from the pool.'), cfg.BoolOpt('memcache_use_advanced_pool', - default=False, + default=True, help='(Optional) Use the advanced (eventlet safe) memcached ' - 'client pool. The advanced pool will only work under ' - 'python 2.x.'), + 'client pool.'), cfg.BoolOpt('include_service_catalog', default=True, help='(Optional) Indicate whether to set the X-Service-Catalog' diff --git a/releasenotes/notes/deprecate-eventlet-unsafe-memcacheclientpool-f8b4a6733513d73e.yaml b/releasenotes/notes/deprecate-eventlet-unsafe-memcacheclientpool-f8b4a6733513d73e.yaml new file mode 100644 index 00000000..22699940 --- /dev/null +++ b/releasenotes/notes/deprecate-eventlet-unsafe-memcacheclientpool-f8b4a6733513d73e.yaml @@ -0,0 +1,22 @@ +--- +deprecations: + - | + We no longer recommend using the eventlet unsafe keystonemiddleware's + memcacheclientpool. This implementation may result in growing connections + to memcached. + + It is recommended that the ``memcache_use_advanced_pool`` option + is set to ``True`` in the ``keystone_authtoken`` configuration section of + the various services (e.g. nova, glance, ...) when memcached is used for + token cache. +upgrade: + - | + [`bug 1892852 `_] + [`bug 1888394 `_] + [`bug 1883659 `_] + Keystonemiddleware now using eventlet-safe implementation of + ``MemcacheClientPool`` from oslo.cache's library by default. + The ``keystonemiddleware`` implementation is now deprecated. For backwards + compatibility, the ``[keystone_authtoken] memcache_use_advanced_pool`` + option can be set to ``False`` config files of the various services (e.g. + nova, glance, ...) when memcached is used for token cache.