Increase httplib._MAXHEADERS

Python 2.7.9+ and 3.2.6+ limits the number of maximum headers in httplib to 100
[1,2,3]. This setting is too low for Swift.

By default the maximum number of allowed headers depends on the number of max
allowed metadata settings plus a default value of 32 for regular http headers.
If for some reason this is not enough (custom middleware for example) it can be
increased with the extra_header_count constraint.

[1] https://bugs.python.org/issue16037
[2] https://hg.python.org/cpython/raw-file/15c95b7d81dc/Misc/NEWS
[3] https://hg.python.org/cpython/raw-file/v3.2.6/Misc/NEWS

Co-Authored-By: Clay Gerrard <clay.gerrard@gmail.com>
Co-Authored-By: Matthew Oliver <matt@oliver.net.au>
Co-Authored-By: Thomas Herve <therve@redhat.com>

Change-Id: I388fd697ec88476024b0e9f1ae75ba35ff765282

etc/swift.conf-sample

@@ -129,6 +129,14 @@ default = yes
 #max_header_size = 8192
 
 
+# By default the maximum number of allowed headers depends on the number of max
+# allowed metadata settings plus a default value of 32 for regular http
+# headers.  If for some reason this is not enough (custom middleware for
+# example) it can be increased with the extra_header_count constraint.
+
+#extra_header_count = 32
+
+
 # max_object_name_length is the max number of bytes in the utf8 encoding
 # of an object name
 

swift/common/bufferedhttp.py

@@ -27,14 +27,19 @@ BufferedHTTPResponse.
 """
 
 from swift import gettext_ as _
+from swift.common import constraints
 from urllib import quote
 import logging
 import time
 import socket
 
+import eventlet
 from eventlet.green.httplib import CONTINUE, HTTPConnection, HTTPMessage, \
     HTTPResponse, HTTPSConnection, _UNKNOWN
 
+httplib = eventlet.import_patched('httplib')
+httplib._MAXHEADERS = constraints.MAX_HEADER_COUNT
+
 
 class BufferedHTTPResponse(HTTPResponse):
     """HTTPResponse class that buffers reading of headers"""

swift/common/constraints.py

@@ -36,6 +36,7 @@ ACCOUNT_LISTING_LIMIT = 10000
 MAX_ACCOUNT_NAME_LENGTH = 256
 MAX_CONTAINER_NAME_LENGTH = 256
 VALID_API_VERSIONS = ["v1", "v1.0"]
+EXTRA_HEADER_COUNT = 0
 
 # If adding an entry to DEFAULT_CONSTRAINTS, note that
 # these constraints are automatically published by the
@@ -54,6 +55,7 @@ DEFAULT_CONSTRAINTS = {
     'max_account_name_length': MAX_ACCOUNT_NAME_LENGTH,
     'max_container_name_length': MAX_CONTAINER_NAME_LENGTH,
     'valid_api_versions': VALID_API_VERSIONS,
+    'extra_header_count': EXTRA_HEADER_COUNT,
 }
 
 SWIFT_CONSTRAINTS_LOADED = False
@@ -105,6 +107,13 @@ FORMAT2CONTENT_TYPE = {'plain': 'text/plain', 'json': 'application/json',
                        'xml': 'application/xml'}
 
 
+# By default the maximum number of allowed headers depends on the number of max
+# allowed metadata settings plus a default value of 32 for regular http
+# headers.  If for some reason this is not enough (custom middleware for
+# example) it can be increased with the extra_header_count constraint.
+MAX_HEADER_COUNT = MAX_META_COUNT + 32 + max(EXTRA_HEADER_COUNT, 0)
+
+
 def check_metadata(req, target_type):
     """
     Check metadata sent in the request headers.  This should only check

test/functional/__init__.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import httplib
 import mock
 import os
 import sys
@@ -52,7 +53,7 @@ from swift.container import server as container_server
 from swift.obj import server as object_server, mem_server as mem_object_server
 import swift.proxy.controllers.obj
 
-
+httplib._MAXHEADERS = constraints.MAX_HEADER_COUNT
 DEBUG = True
 
 # In order to get the proper blocking behavior of sockets without using

test/functional/swift_test_client.py

@@ -29,10 +29,13 @@ from xml.dom import minidom
 
 from swiftclient import get_auth
 
+from swift.common import constraints
 from swift.common.utils import config_true_value
 
 from test import safe_repr
 
+httplib._MAXHEADERS = constraints.MAX_HEADER_COUNT
+
 
 class AuthenticationFailed(Exception):
     pass

test/unit/proxy/test_server.py

@@ -9168,7 +9168,7 @@ class TestSwiftInfo(unittest.TestCase):
                          constraints.VALID_API_VERSIONS)
         # this next test is deliberately brittle in order to alert if
         # other items are added to swift info
-        self.assertEqual(len(si), 17)
+        self.assertEqual(len(si), 18)
 
         self.assertTrue('policies' in si)
         sorted_pols = sorted(si['policies'], key=operator.itemgetter('name'))