From 023a0615874443e2fc28d0df64a5ab5dd485405d Mon Sep 17 00:00:00 2001 From: Peter Portante Date: Sun, 18 Aug 2013 13:37:44 -0400 Subject: [PATCH] Tie socket write buffer size to server parameters By default, Python 2.*'s standard library "socket" module performs 8K writes. For 10ge networks, with large MTUs (typically 9,000), this is not optimal. We tie the default buffer size to the client_chunk_size paramter for the proxy server, and to the network_chunk_size for the object server. One might be tempted to ask, isn't there a way to set this value on a per-request basis? This author was unable to find a reference to the _fileobject in the context of WSGI. By the time a request pass to a WSGI object's __call__ method, the "wfile" attribute of the req.environ['eventlet.input'] (Input) object has been set to None, and the "rfile" attribute is the object wrapping the socket for reading, not writing. One might also be tempted to ask, why not just override the wsgi.HttpProtocol's "wbufsize" class attribute instead? Until eventlet/wsgi.py is fixed, we can't set wsgi.HttpProtocol.wbufsize to anything but zero (the default, see Python's SocketServer.py, StreamRequestHandler class), since Eventlet does not ensure the socket _fileobject's flush() method is called after Eventlet invokes a write() method on the same. NOTE: wbufsize (a class attribute of StreamRequestHandler originally, not to be confused with the standard library's socket._fileobject._wbufsize class attribute) is used for the bufsize parameter of the connection object's makefile() method. As a result, the socket's _fileobject code uses that value to set both _rbufsize and _wbufsize. While that would allow us to transmit in 64KB chunks, it also means that write() and writeline() method calls on the socket _fileobject are only transmitted once 64KB have been accumulated, or a flush() is called. As for performance improvement: Run 8KB 64KB 0 8.101 6.367 1 7.892 6.216 2 7.732 6.246 3 7.594 6.229 4 7.594 6.292 5 7.555 6.230 6 7.575 6.270 7 7.528 6.278 8 7.547 6.304 9 7.550 6.313 Average 7.667 6.275 1.3923 18.16% Run using the following after adjusting the test value for obj_len to 1 GB: nosetests -v --nocapture --nologcapture \ test/unit/proxy/test_server.py:TestProxyObjectPerformance.test_GET_debug_large_file Change-Id: I4dd93acc3376e9960fbdcdcae00c6d002e545894 Signed-off-by: Peter Portante --- swift/obj/server.py | 16 +++++++++ swift/proxy/server.py | 13 ++++++++ test/unit/proxy/test_server.py | 59 ++++++++++++++++++++++++++++++++++ 3 files changed, 88 insertions(+) diff --git a/swift/obj/server.py b/swift/obj/server.py index e8593fedab..0e902183a7 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -20,6 +20,7 @@ import cPickle as pickle import os import time import traceback +import socket from datetime import datetime from swift import gettext_ as _ from hashlib import md5 @@ -90,6 +91,21 @@ class ObjectController(object): 'expiring_objects' self.expiring_objects_container_divisor = \ int(conf.get('expiring_objects_container_divisor') or 86400) + # Initialization was successful, so now apply the network chunk size + # parameter as the default read / write buffer size for the network + # sockets. + # + # NOTE WELL: This is a class setting, so until we get set this on a + # per-connection basis, this affects reading and writing on ALL + # sockets, those between the proxy servers and external clients, and + # those between the proxy servers and the other internal servers. + # + # ** Because the primary motivation for this is to optimize how data + # is written back to the proxy server, we could use the value from the + # disk_chunk_size parameter. However, it affects all created sockets + # using this class so we have chosen to tie it to the + # network_chunk_size parameter value instead. + socket._fileobject.default_bufsize = self.network_chunk_size # Provide further setup sepecific to an object server implemenation. self.setup(conf) diff --git a/swift/proxy/server.py b/swift/proxy/server.py index bbc439f5b6..bcd0eb562c 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -26,6 +26,7 @@ import mimetypes import os +import socket from swift import gettext_ as _ from random import shuffle from time import time @@ -160,6 +161,18 @@ class Application(object): self.swift_owner_headers = [ name.strip() for name in swift_owner_headers.split(',') if name.strip()] + # Initialization was successful, so now apply the client chunk size + # parameter as the default read / write buffer size for the network + # sockets. + # + # NOTE WELL: This is a class setting, so until we get set this on a + # per-connection basis, this affects reading and writing on ALL + # sockets, those between the proxy servers and external clients, and + # those between the proxy servers and the other internal servers. + # + # ** Because it affects the client as well, currently, we use the + # client chunk size as the govenor and not the object chunk size. + socket._fileobject.default_bufsize = self.client_chunk_size def get_controller(self, path): """ diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 071f62ae11..a16bf219e5 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -6716,6 +6716,65 @@ class TestSegmentedIterable(unittest.TestCase): self.assertEquals(''.join(segit.app_iter_range(5, 7)), '34') +class TestProxyObjectPerformance(unittest.TestCase): + + def setUp(self): + # This is just a simple test that can be used to verify and debug the + # various data paths between the proxy server and the object + # server. Used as a play ground to debug buffer sizes for sockets. + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + # Client is transmitting in 2 MB chunks + fd = sock.makefile('wb', 2 * 1024 * 1024) + # Small, fast for testing + obj_len = 2 * 64 * 1024 + # Use 1 GB or more for measurements + #obj_len = 2 * 512 * 1024 * 1024 + self.path = '/v1/a/c/o.large' + fd.write('PUT %s HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'X-Storage-Token: t\r\n' + 'Content-Length: %s\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n' % (self.path, str(obj_len))) + fd.write('a' * obj_len) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + self.obj_len = obj_len + + def test_GET_debug_large_file(self): + for i in range(0, 10): + start = time.time() + + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + # Client is reading in 2 MB chunks + fd = sock.makefile('wb', 2 * 1024 * 1024) + fd.write('GET %s HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'X-Storage-Token: t\r\n' + '\r\n' % self.path) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 200' + self.assertEqual(headers[:len(exp)], exp) + + total = 0 + while True: + buf = fd.read(100000) + if not buf: + break + total += len(buf) + self.assertEqual(total, self.obj_len) + + end = time.time() + print "Run %02d took %07.03f" % (i, end - start) + + if __name__ == '__main__': setup() try: