diff --git a/swift/obj/server.py b/swift/obj/server.py index e8593fedab..0e902183a7 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -20,6 +20,7 @@ import cPickle as pickle import os import time import traceback +import socket from datetime import datetime from swift import gettext_ as _ from hashlib import md5 @@ -90,6 +91,21 @@ class ObjectController(object): 'expiring_objects' self.expiring_objects_container_divisor = \ int(conf.get('expiring_objects_container_divisor') or 86400) + # Initialization was successful, so now apply the network chunk size + # parameter as the default read / write buffer size for the network + # sockets. + # + # NOTE WELL: This is a class setting, so until we get set this on a + # per-connection basis, this affects reading and writing on ALL + # sockets, those between the proxy servers and external clients, and + # those between the proxy servers and the other internal servers. + # + # ** Because the primary motivation for this is to optimize how data + # is written back to the proxy server, we could use the value from the + # disk_chunk_size parameter. However, it affects all created sockets + # using this class so we have chosen to tie it to the + # network_chunk_size parameter value instead. + socket._fileobject.default_bufsize = self.network_chunk_size # Provide further setup sepecific to an object server implemenation. self.setup(conf) diff --git a/swift/proxy/server.py b/swift/proxy/server.py index bbc439f5b6..bcd0eb562c 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -26,6 +26,7 @@ import mimetypes import os +import socket from swift import gettext_ as _ from random import shuffle from time import time @@ -160,6 +161,18 @@ class Application(object): self.swift_owner_headers = [ name.strip() for name in swift_owner_headers.split(',') if name.strip()] + # Initialization was successful, so now apply the client chunk size + # parameter as the default read / write buffer size for the network + # sockets. + # + # NOTE WELL: This is a class setting, so until we get set this on a + # per-connection basis, this affects reading and writing on ALL + # sockets, those between the proxy servers and external clients, and + # those between the proxy servers and the other internal servers. + # + # ** Because it affects the client as well, currently, we use the + # client chunk size as the govenor and not the object chunk size. + socket._fileobject.default_bufsize = self.client_chunk_size def get_controller(self, path): """ diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 071f62ae11..a16bf219e5 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -6716,6 +6716,65 @@ class TestSegmentedIterable(unittest.TestCase): self.assertEquals(''.join(segit.app_iter_range(5, 7)), '34') +class TestProxyObjectPerformance(unittest.TestCase): + + def setUp(self): + # This is just a simple test that can be used to verify and debug the + # various data paths between the proxy server and the object + # server. Used as a play ground to debug buffer sizes for sockets. + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + # Client is transmitting in 2 MB chunks + fd = sock.makefile('wb', 2 * 1024 * 1024) + # Small, fast for testing + obj_len = 2 * 64 * 1024 + # Use 1 GB or more for measurements + #obj_len = 2 * 512 * 1024 * 1024 + self.path = '/v1/a/c/o.large' + fd.write('PUT %s HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'X-Storage-Token: t\r\n' + 'Content-Length: %s\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n' % (self.path, str(obj_len))) + fd.write('a' * obj_len) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + self.obj_len = obj_len + + def test_GET_debug_large_file(self): + for i in range(0, 10): + start = time.time() + + prolis = _test_sockets[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + # Client is reading in 2 MB chunks + fd = sock.makefile('wb', 2 * 1024 * 1024) + fd.write('GET %s HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'X-Storage-Token: t\r\n' + '\r\n' % self.path) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 200' + self.assertEqual(headers[:len(exp)], exp) + + total = 0 + while True: + buf = fd.read(100000) + if not buf: + break + total += len(buf) + self.assertEqual(total, self.obj_len) + + end = time.time() + print "Run %02d took %07.03f" % (i, end - start) + + if __name__ == '__main__': setup() try: