From edfca861b6fa39972df276fb1f37aa81583a605d Mon Sep 17 00:00:00 2001 From: Christian Schwede Date: Fri, 26 Jun 2015 08:54:22 +0000 Subject: [PATCH] Increase httplib._MAXHEADERS Python 2.7.9+ and 3.2.6+ limits the number of maximum headers in httplib to 100 [1,2,3]. This setting is too low for Swift. By default the maximum number of allowed headers depends on the number of max allowed metadata settings plus a default value of 32 for regular http headers. If for some reason this is not enough (custom middleware for example) it can be increased with the extra_header_count constraint. [1] https://bugs.python.org/issue16037 [2] https://hg.python.org/cpython/raw-file/15c95b7d81dc/Misc/NEWS [3] https://hg.python.org/cpython/raw-file/v3.2.6/Misc/NEWS Co-Authored-By: Clay Gerrard Co-Authored-By: Matthew Oliver Co-Authored-By: Thomas Herve Change-Id: I388fd697ec88476024b0e9f1ae75ba35ff765282 --- etc/swift.conf-sample | 8 ++++++++ swift/common/bufferedhttp.py | 5 +++++ swift/common/constraints.py | 9 +++++++++ test/functional/__init__.py | 3 ++- test/functional/swift_test_client.py | 3 +++ test/unit/proxy/test_server.py | 2 +- 6 files changed, 28 insertions(+), 2 deletions(-) diff --git a/etc/swift.conf-sample b/etc/swift.conf-sample index f8accabaec..76d1e876ae 100644 --- a/etc/swift.conf-sample +++ b/etc/swift.conf-sample @@ -129,6 +129,14 @@ default = yes #max_header_size = 8192 +# By default the maximum number of allowed headers depends on the number of max +# allowed metadata settings plus a default value of 32 for regular http +# headers. If for some reason this is not enough (custom middleware for +# example) it can be increased with the extra_header_count constraint. + +#extra_header_count = 32 + + # max_object_name_length is the max number of bytes in the utf8 encoding # of an object name diff --git a/swift/common/bufferedhttp.py b/swift/common/bufferedhttp.py index c7acccc27c..528ac214cf 100644 --- a/swift/common/bufferedhttp.py +++ b/swift/common/bufferedhttp.py @@ -27,14 +27,19 @@ BufferedHTTPResponse. """ from swift import gettext_ as _ +from swift.common import constraints from urllib import quote import logging import time import socket +import eventlet from eventlet.green.httplib import CONTINUE, HTTPConnection, HTTPMessage, \ HTTPResponse, HTTPSConnection, _UNKNOWN +httplib = eventlet.import_patched('httplib') +httplib._MAXHEADERS = constraints.MAX_HEADER_COUNT + class BufferedHTTPResponse(HTTPResponse): """HTTPResponse class that buffers reading of headers""" diff --git a/swift/common/constraints.py b/swift/common/constraints.py index 4cee56ab3c..591168b5e9 100644 --- a/swift/common/constraints.py +++ b/swift/common/constraints.py @@ -36,6 +36,7 @@ ACCOUNT_LISTING_LIMIT = 10000 MAX_ACCOUNT_NAME_LENGTH = 256 MAX_CONTAINER_NAME_LENGTH = 256 VALID_API_VERSIONS = ["v1", "v1.0"] +EXTRA_HEADER_COUNT = 0 # If adding an entry to DEFAULT_CONSTRAINTS, note that # these constraints are automatically published by the @@ -54,6 +55,7 @@ DEFAULT_CONSTRAINTS = { 'max_account_name_length': MAX_ACCOUNT_NAME_LENGTH, 'max_container_name_length': MAX_CONTAINER_NAME_LENGTH, 'valid_api_versions': VALID_API_VERSIONS, + 'extra_header_count': EXTRA_HEADER_COUNT, } SWIFT_CONSTRAINTS_LOADED = False @@ -105,6 +107,13 @@ FORMAT2CONTENT_TYPE = {'plain': 'text/plain', 'json': 'application/json', 'xml': 'application/xml'} +# By default the maximum number of allowed headers depends on the number of max +# allowed metadata settings plus a default value of 32 for regular http +# headers. If for some reason this is not enough (custom middleware for +# example) it can be increased with the extra_header_count constraint. +MAX_HEADER_COUNT = MAX_META_COUNT + 32 + max(EXTRA_HEADER_COUNT, 0) + + def check_metadata(req, target_type): """ Check metadata sent in the request headers. This should only check diff --git a/test/functional/__init__.py b/test/functional/__init__.py index 73e5006638..580de56c81 100644 --- a/test/functional/__init__.py +++ b/test/functional/__init__.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import httplib import mock import os import sys @@ -52,7 +53,7 @@ from swift.container import server as container_server from swift.obj import server as object_server, mem_server as mem_object_server import swift.proxy.controllers.obj - +httplib._MAXHEADERS = constraints.MAX_HEADER_COUNT DEBUG = True # In order to get the proper blocking behavior of sockets without using diff --git a/test/functional/swift_test_client.py b/test/functional/swift_test_client.py index 695ea202d7..8767e62476 100644 --- a/test/functional/swift_test_client.py +++ b/test/functional/swift_test_client.py @@ -29,10 +29,13 @@ from xml.dom import minidom from swiftclient import get_auth +from swift.common import constraints from swift.common.utils import config_true_value from test import safe_repr +httplib._MAXHEADERS = constraints.MAX_HEADER_COUNT + class AuthenticationFailed(Exception): pass diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 46370590eb..500ab6105a 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -9168,7 +9168,7 @@ class TestSwiftInfo(unittest.TestCase): constraints.VALID_API_VERSIONS) # this next test is deliberately brittle in order to alert if # other items are added to swift info - self.assertEqual(len(si), 17) + self.assertEqual(len(si), 18) self.assertTrue('policies' in si) sorted_pols = sorted(si['policies'], key=operator.itemgetter('name'))