Tie socket write buffer size to server parameters

By default, Python 2.*'s standard library "socket" module performs 8K
writes. For 10ge networks, with large MTUs (typically 9,000), this is
not optimal. We tie the default buffer size to the client_chunk_size
paramter for the proxy server, and to the network_chunk_size for the
object server.

One might be tempted to ask, isn't there a way to set this value on a
per-request basis? This author was unable to find a reference to the
_fileobject in the context of WSGI. By the time a request pass to a
WSGI object's __call__ method, the "wfile" attribute of the
req.environ['eventlet.input'] (Input) object has been set to None, and
the "rfile" attribute is the object wrapping the socket for reading,
not writing.

One might also be tempted to ask, why not just override the
wsgi.HttpProtocol's "wbufsize" class attribute instead? Until
eventlet/wsgi.py is fixed, we can't set wsgi.HttpProtocol.wbufsize to
anything but zero (the default, see Python's SocketServer.py,
StreamRequestHandler class), since Eventlet does not ensure the socket
_fileobject's flush() method is called after Eventlet invokes a
write() method on the same.  NOTE: wbufsize (a class attribute of
StreamRequestHandler originally, not to be confused with the standard
library's socket._fileobject._wbufsize class attribute) is used for
the bufsize parameter of the connection object's makefile() method. As
a result, the socket's _fileobject code uses that value to set both
_rbufsize and _wbufsize. While that would allow us to transmit in 64KB
chunks, it also means that write() and writeline() method calls on the
socket _fileobject are only transmitted once 64KB have been
accumulated, or a flush() is called.

As for performance improvement:

Run       8KB   64KB
  0     8.101  6.367
  1     7.892  6.216
  2     7.732  6.246
  3     7.594  6.229
  4     7.594  6.292
  5     7.555  6.230
  6     7.575  6.270
  7     7.528  6.278
  8     7.547  6.304
  9     7.550  6.313
Average 7.667  6.275  1.3923  18.16%

Run using the following after adjusting the test value for obj_len to
1 GB:

nosetests -v --nocapture --nologcapture \
test/unit/proxy/test_server.py:TestProxyObjectPerformance.test_GET_debug_large_file

Change-Id: I4dd93acc3376e9960fbdcdcae00c6d002e545894
Signed-off-by: Peter Portante <peter.portante@redhat.com>

swift/obj/server.py

@@ -20,6 +20,7 @@ import cPickle as pickle
 import os
 import time
 import traceback
+import socket
 from datetime import datetime
 from swift import gettext_ as _
 from hashlib import md5
@@ -90,6 +91,21 @@ class ObjectController(object):
             'expiring_objects'
         self.expiring_objects_container_divisor = \
             int(conf.get('expiring_objects_container_divisor') or 86400)
+        # Initialization was successful, so now apply the network chunk size
+        # parameter as the default read / write buffer size for the network
+        # sockets.
+        #
+        # NOTE WELL: This is a class setting, so until we get set this on a
+        # per-connection basis, this affects reading and writing on ALL
+        # sockets, those between the proxy servers and external clients, and
+        # those between the proxy servers and the other internal servers.
+        #
+        # ** Because the primary motivation for this is to optimize how data
+        # is written back to the proxy server, we could use the value from the
+        # disk_chunk_size parameter. However, it affects all created sockets
+        # using this class so we have chosen to tie it to the
+        # network_chunk_size parameter value instead.
+        socket._fileobject.default_bufsize = self.network_chunk_size
 
         # Provide further setup sepecific to an object server implemenation.
         self.setup(conf)

swift/proxy/server.py

@@ -26,6 +26,7 @@
 
 import mimetypes
 import os
+import socket
 from swift import gettext_ as _
 from random import shuffle
 from time import time
@@ -160,6 +161,18 @@ class Application(object):
         self.swift_owner_headers = [
             name.strip()
             for name in swift_owner_headers.split(',') if name.strip()]
+        # Initialization was successful, so now apply the client chunk size
+        # parameter as the default read / write buffer size for the network
+        # sockets.
+        #
+        # NOTE WELL: This is a class setting, so until we get set this on a
+        # per-connection basis, this affects reading and writing on ALL
+        # sockets, those between the proxy servers and external clients, and
+        # those between the proxy servers and the other internal servers.
+        #
+        # ** Because it affects the client as well, currently, we use the
+        # client chunk size as the govenor and not the object chunk size.
+        socket._fileobject.default_bufsize = self.client_chunk_size
 
     def get_controller(self, path):
         """

test/unit/proxy/test_server.py

@@ -6716,6 +6716,65 @@ class TestSegmentedIterable(unittest.TestCase):
         self.assertEquals(''.join(segit.app_iter_range(5, 7)), '34')
 
 
+class TestProxyObjectPerformance(unittest.TestCase):
+
+    def setUp(self):
+        # This is just a simple test that can be used to verify and debug the
+        # various data paths between the proxy server and the object
+        # server. Used as a play ground to debug buffer sizes for sockets.
+        prolis = _test_sockets[0]
+        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
+        # Client is transmitting in 2 MB chunks
+        fd = sock.makefile('wb', 2 * 1024 * 1024)
+        # Small, fast for testing
+        obj_len = 2 * 64 * 1024
+        # Use 1 GB or more for measurements
+        #obj_len = 2 * 512 * 1024 * 1024
+        self.path = '/v1/a/c/o.large'
+        fd.write('PUT %s HTTP/1.1\r\n'
+                 'Host: localhost\r\n'
+                 'Connection: close\r\n'
+                 'X-Storage-Token: t\r\n'
+                 'Content-Length: %s\r\n'
+                 'Content-Type: application/octet-stream\r\n'
+                 '\r\n' % (self.path, str(obj_len)))
+        fd.write('a' * obj_len)
+        fd.flush()
+        headers = readuntil2crlfs(fd)
+        exp = 'HTTP/1.1 201'
+        self.assertEqual(headers[:len(exp)], exp)
+        self.obj_len = obj_len
+
+    def test_GET_debug_large_file(self):
+        for i in range(0, 10):
+            start = time.time()
+
+            prolis = _test_sockets[0]
+            sock = connect_tcp(('localhost', prolis.getsockname()[1]))
+            # Client is reading in 2 MB chunks
+            fd = sock.makefile('wb', 2 * 1024 * 1024)
+            fd.write('GET %s HTTP/1.1\r\n'
+                     'Host: localhost\r\n'
+                     'Connection: close\r\n'
+                     'X-Storage-Token: t\r\n'
+                     '\r\n' % self.path)
+            fd.flush()
+            headers = readuntil2crlfs(fd)
+            exp = 'HTTP/1.1 200'
+            self.assertEqual(headers[:len(exp)], exp)
+
+            total = 0
+            while True:
+                buf = fd.read(100000)
+                if not buf:
+                    break
+                total += len(buf)
+            self.assertEqual(total, self.obj_len)
+
+            end = time.time()
+            print "Run %02d took %07.03f" % (i, end - start)
+
+
 if __name__ == '__main__':
     setup()
     try: