From c0bf01afdb88425f4cd5b504fd2cb49b3f23e824 Mon Sep 17 00:00:00 2001 From: David Goetz Date: Mon, 27 Jan 2014 15:32:01 -0800 Subject: [PATCH] Config option to lower the timeout for recoverable object GETs. Change-Id: I71f9824559126e4025e7629715ab9dac64231e09 --- doc/source/deployment_guide.rst | 4 ++++ etc/proxy-server.conf-sample | 13 +++++++++++++ swift/proxy/controllers/base.py | 15 ++++++++++----- swift/proxy/server.py | 2 ++ test/unit/proxy/test_server.py | 4 ++-- 5 files changed, 31 insertions(+), 7 deletions(-) diff --git a/doc/source/deployment_guide.rst b/doc/source/deployment_guide.rst index b6d74351d6..65ee6aaaec 100644 --- a/doc/source/deployment_guide.rst +++ b/doc/source/deployment_guide.rst @@ -875,6 +875,10 @@ memcache_max_connections 2 Max number of connections to worker node_timeout 10 Request timeout to external services +recoverable_node_timeout node_timeout Request timeout to external + services for requests that, on + failure, can be recovered + from. For example, object GET. client_timeout 60 Timeout to read one chunk from a client conn_timeout 0.5 Connection timeout to diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 0de120fd6e..e8a1209865 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -84,7 +84,20 @@ use = egg:swift#proxy # recheck_container_existence = 60 # object_chunk_size = 8192 # client_chunk_size = 8192 +# +# How long the proxy server will wait on responses from the a/c/o servers. # node_timeout = 10 +# +# How long the proxy server will wait for an initial response and to read a +# chunk of data from the object servers while serving GET / HEAD requests. +# Timeouts from these requests can be recovered from so setting this to +# something lower than node_timeout would provide quicker error recovery +# while allowing for a longer timeout for non-recoverable requests (PUTs). +# Defaults to node_timeout, should be overriden if node_timeout is set to a +# high number to prevent client timeouts from firing before the proxy server +# has a chance to retry. +# recoverable_node_timeout = node_timeout +# # conn_timeout = 0.5 # # How long to wait for requests to finish after a quorum has been established. diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index 403ae1df28..44e944c654 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -654,9 +654,12 @@ class GetOrHeadHandler(object): try: nchunks = 0 bytes_read_from_source = 0 + node_timeout = self.app.node_timeout + if self.server_type == 'Object': + node_timeout = self.app.recoverable_node_timeout while True: try: - with ChunkReadTimeout(self.app.node_timeout): + with ChunkReadTimeout(node_timeout): chunk = source.read(self.app.object_chunk_size) nchunks += 1 bytes_read_from_source += len(chunk) @@ -724,13 +727,15 @@ class GetOrHeadHandler(object): close_swift_conn(source) def _get_source_and_node(self): - self.statuses = [] self.reasons = [] self.bodies = [] self.source_headers = [] sources = [] + node_timeout = self.app.node_timeout + if self.server_type == 'Object' and not self.newest: + node_timeout = self.app.recoverable_node_timeout for node in self.app.iter_nodes(self.ring, self.partition): if node in self.used_nodes: continue @@ -744,7 +749,7 @@ class GetOrHeadHandler(object): query_string=self.req_query_string) self.app.set_node_timing(node, time.time() - start_node_timing) - with Timeout(self.app.node_timeout): + with Timeout(node_timeout): possible_source = conn.getresponse() # See NOTE: swift_conn at top of file about this. possible_source.swift_conn = conn @@ -1155,7 +1160,7 @@ class Controller(object): Base handler for HTTP GET or HEAD requests. :param req: swob.Request object - :param server_type: server type + :param server_type: server type used in logging :param ring: the ring to obtain nodes from :param partition: partition :param path: path for the request @@ -1164,7 +1169,7 @@ class Controller(object): backend_headers = self.generate_request_headers( req, additional=req.headers) - handler = GetOrHeadHandler(self.app, req, server_type, ring, + handler = GetOrHeadHandler(self.app, req, self.server_type, ring, partition, path, backend_headers) res = handler.get_working_response(req) diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 4701f44664..c39d98aafd 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -78,6 +78,8 @@ class Application(object): swift_dir = conf.get('swift_dir', '/etc/swift') self.node_timeout = int(conf.get('node_timeout', 10)) + self.recoverable_node_timeout = int( + conf.get('recoverable_node_timeout', self.node_timeout)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.client_timeout = int(conf.get('client_timeout', 60)) self.put_queue_depth = int(conf.get('put_queue_depth', 10)) diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 50e951cc7b..e8d4ce8911 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -1697,7 +1697,7 @@ class TestObjectController(unittest.TestCase): except ChunkReadTimeout: got_exc = True self.assert_(not got_exc) - self.app.node_timeout = 0.1 + self.app.recoverable_node_timeout = 0.1 set_http_connect(200, 200, 200, slow=True) resp = req.get_response(self.app) got_exc = False @@ -1712,7 +1712,7 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) self.app.update_request(req) - self.app.node_timeout = 0.1 + self.app.recoverable_node_timeout = 0.1 set_http_connect(200, 200, 200, slow=[3]) resp = req.get_response(self.app) got_exc = False