From d9c4913e3b1aaba378d4786ddfefa6265c195f71 Mon Sep 17 00:00:00 2001 From: Samuel Merritt Date: Mon, 7 Mar 2016 18:18:35 -0800 Subject: [PATCH] Make eventlet.tpool's thread count configurable in object server If you're running servers_per_port > 0 and threads_per_disk = 0 (as it should be with servers_per_port on), each object-server process will have 20 IO threads waiting around to service eventlet.tpool calls. This is far too many; with servers_per_port, there's no real benefit to having so many IO threads. This commit makes it so that, when servers_per_port > 0, each object server defaults to having one main thread and one IO thread. Also, eventlet's tpool size is now configurable via the object-server config file. If a tpool size is set, that's what we'll use regardless of servers_per_port. This allows operators with an excess of threads to remove some regardless of servers_per_port. Change-Id: I8f8914b7e70f2510393eb7c5e6be9708631ac027 Closes-Bug: 1554233 --- doc/source/deployment_guide.rst | 13 +++++++++++++ etc/object-server.conf-sample | 23 +++++++++++++++++++++++ swift/obj/server.py | 33 +++++++++++++++++++++++++++++++-- test/unit/obj/test_server.py | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 100 insertions(+), 2 deletions(-) diff --git a/doc/source/deployment_guide.rst b/doc/source/deployment_guide.rst index 8bf7e87ca1..058a04e5af 100644 --- a/doc/source/deployment_guide.rst +++ b/doc/source/deployment_guide.rst @@ -651,6 +651,19 @@ ionice_priority None I/O scheduling priority of priority of the process. Work only with ionice_class. Ignored if IOPRIO_CLASS_IDLE is set. +eventlet_tpool_num_threads auto The number of threads in eventlet's thread pool. + Most IO will occur in the object server's main + thread, but certain "heavy" IO operations will + occur in separate IO threads, managed by + eventlet. + The default value is auto, whose actual value + is dependant on the servers_per_port value. + If servers_per_port is zero then it uses + eventlet's default (currently 20 threads). + If the servers_per_port is nonzero then it'll + only use 1 thread per process. + This value can be overridden with an integer + value. ============================= ====================== =============================================== [object-replicator] diff --git a/etc/object-server.conf-sample b/etc/object-server.conf-sample index f03e5af1d0..626c260fa4 100644 --- a/etc/object-server.conf-sample +++ b/etc/object-server.conf-sample @@ -124,6 +124,29 @@ use = egg:swift#object # # auto_create_account_prefix = . # + +# The number of threads in eventlet's thread pool. Most IO will occur +# in the object server's main thread, but certain "heavy" IO +# operations will occur in separate IO threads, managed by eventlet. +# +# The default value is auto, whose actual value is dependant on the +# servers_per_port value: +# +# - When servers_per_port is zero, the default value of +# eventlet_tpool_num_threads is empty, which uses eventlet's default +# (currently 20 threads). +# +# - When servers_per_port is nonzero, the default value of +# eventlet_tpool_num_threads is 1. +# +# But you may override this value to any integer value. +# +# Note that this value is threads per object-server process, so to +# compute the total number of IO threads on a node, you must multiply +# this by the number of object-server processes on the node. +# +# eventlet_tpool_num_threads = auto + # Configure parameter for creating specific server # To handle all verbs, including replication verbs, do not specify # "replication_server" (this is the default). To only handle replication, diff --git a/swift/obj/server.py b/swift/obj/server.py index 1efa3997c1..e33adb8126 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -26,14 +26,15 @@ import math from swift import gettext_ as _ from hashlib import md5 -from eventlet import sleep, wsgi, Timeout +from eventlet import sleep, wsgi, Timeout, tpool from eventlet.greenthread import spawn from swift.common.utils import public, get_logger, \ config_true_value, timing_stats, replication, \ normalize_delete_at_timestamp, get_log_line, Timestamp, \ get_expirer_container, parse_mime_headers, \ - iter_multipart_mime_documents, extract_swift_bytes, safe_json_loads + iter_multipart_mime_documents, extract_swift_bytes, safe_json_loads, \ + config_auto_int_value from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_object_creation, \ valid_timestamp, check_utf8 @@ -198,6 +199,34 @@ class ObjectController(BaseStorageServer): self.replication_failure_ratio = float( conf.get('replication_failure_ratio') or 1.0) + servers_per_port = int(conf.get('servers_per_port', '0') or 0) + if servers_per_port: + # The typical servers-per-port deployment also uses one port per + # disk, so you really get N servers per disk. In that case, + # having a pool of 20 threads per server per disk is far too + # much. For example, given a 60-disk chassis and 4 servers per + # disk, the default configuration will give us 21 threads per + # server (the main thread plus the twenty tpool threads), for a + # total of around 60 * 21 * 4 = 5040 threads. This is clearly + # too high. + # + # Instead, we use a tpool size of 1, giving us 2 threads per + # process. In the example above, that's 60 * 2 * 4 = 480 + # threads, which is reasonable since there are 240 processes. + default_tpool_size = 1 + else: + # If we're not using servers-per-port, then leave the tpool size + # alone. The default (20) is typically good enough for one + # object server handling requests for many disks. + default_tpool_size = None + + tpool_size = config_auto_int_value( + conf.get('eventlet_tpool_num_threads'), + default_tpool_size) + + if tpool_size: + tpool.set_num_threads(tpool_size) + def get_diskfile(self, device, partition, account, container, obj, policy, **kwargs): """ diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index 11d5fedf56..34debeb876 100644 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -101,6 +101,39 @@ def fake_spawn(): gt.wait() +class TestTpoolSize(unittest.TestCase): + def test_default_config(self): + with mock.patch('eventlet.tpool.set_num_threads') as mock_snt: + object_server.ObjectController({}) + self.assertEqual([], mock_snt.mock_calls) + + def test_explicit_setting(self): + conf = {'eventlet_tpool_num_threads': '17'} + with mock.patch('eventlet.tpool.set_num_threads') as mock_snt: + object_server.ObjectController(conf) + self.assertEqual([mock.call(17)], mock_snt.mock_calls) + + def test_servers_per_port_no_explicit_setting(self): + conf = {'servers_per_port': '3'} + with mock.patch('eventlet.tpool.set_num_threads') as mock_snt: + object_server.ObjectController(conf) + self.assertEqual([mock.call(1)], mock_snt.mock_calls) + + def test_servers_per_port_with_explicit_setting(self): + conf = {'eventlet_tpool_num_threads': '17', + 'servers_per_port': '3'} + with mock.patch('eventlet.tpool.set_num_threads') as mock_snt: + object_server.ObjectController(conf) + self.assertEqual([mock.call(17)], mock_snt.mock_calls) + + def test_servers_per_port_empty(self): + # run_wsgi is robust to this, so we should be too + conf = {'servers_per_port': ''} + with mock.patch('eventlet.tpool.set_num_threads') as mock_snt: + object_server.ObjectController(conf) + self.assertEqual([], mock_snt.mock_calls) + + @patch_policies(test_policies) class TestObjectController(unittest.TestCase): """Test swift.obj.server.ObjectController"""