From bae5561d4d3ded0a7c22f4a3e90b2a8e535a5384 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Mon, 6 Nov 2023 10:55:45 +0100 Subject: [PATCH] Clean-up memcache connection sockets The ConnectionPool currently opens sockets, but never closes them. As a result, any client using oslo.cache memcache pool leaves sockets in a CLOSE_WAIT state, with the source port not being re-usable. In our production system, were we have A LOT of activity, this creates a storm of non-reusable ports: all source ports are in use, making the node unuseable for other things. This patch adds a __del__ destructor closing inactive connections, fixing the issue. Closes-Bug: #2043121 Change-Id: I09d632346c76d1aff7c534f0d040162d1985f548 --- oslo_cache/_memcache_pool.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/oslo_cache/_memcache_pool.py b/oslo_cache/_memcache_pool.py index 21e1e0dd..57fe47bb 100644 --- a/oslo_cache/_memcache_pool.py +++ b/oslo_cache/_memcache_pool.py @@ -93,6 +93,27 @@ class ConnectionPool(queue.Queue): self._connection_get_timeout = conn_get_timeout self._acquired = 0 + def __del__(self): + """Delete the connection pool. + + Destory all connections left in the queue. + """ + while True: + # As per https://docs.python.org/3/library/collections.html + # self.queue.pop() will raise IndexError when no elements are + # present, ending the while True: loop. + # The logic loops over all connections in the queue but it does + # not retry for a single one in case a connection closure fails + # then it leaves that one and process the next. + try: + conn = self.queue.pop().connection + self._destroy_connection(conn) + except IndexError: + break + except Exception as e: + self._do_log( + LOG.warning, "Unable to cleanup a connection: %s", e) + def _create_connection(self): """Returns a connection instance.