openstack/swift
Code Issues Proposed changes

Change chunks_per_sync config to mb_per_sync

Browse Source
This commit is contained in:
Michael Barton 2010-10-13 21:26:43 +00:00
parent a938d2c6ab
commit c27da7bb9d
3 changed files with 7 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
etc/object-server.conf-sample
View File

@ -20,6 +20,7 @@ use = egg:swift#object
# conn_timeout = 0.5 # conn_timeout = 0.5
# network_chunk_size = 65536 # network_chunk_size = 65536
# disk_chunk_size = 65536 # disk_chunk_size = 65536
# mb_per_sync = 512
# max_upload_time = 86400 # max_upload_time = 86400
# slow = 1 # slow = 1

13
swift/obj/server.py
View File

@ -259,7 +259,7 @@ class ObjectController(object):
self.log_requests = conf.get('log_requests', 't')[:1].lower() == 't' self.log_requests = conf.get('log_requests', 't')[:1].lower() == 't'
self.max_upload_time = int(conf.get('max_upload_time', 86400)) self.max_upload_time = int(conf.get('max_upload_time', 86400))
self.slow = int(conf.get('slow', 0)) self.slow = int(conf.get('slow', 0))
self.chunks_per_sync = int(conf.get('chunks_per_sync', 8000)) self.bytes_per_sync = int(conf.get('mb_per_sync', 512) * 1024 * 1024)
def container_update(self, op, account, container, obj, headers_in, def container_update(self, op, account, container, obj, headers_in,
headers_out, objdevice): headers_out, objdevice):
@ -359,11 +359,10 @@ class ObjectController(object):
upload_expiration = time.time() + self.max_upload_time upload_expiration = time.time() + self.max_upload_time
etag = md5() etag = md5()
upload_size = 0 upload_size = 0
last_sync = 0
with file.mkstemp() as (fd, tmppath): with file.mkstemp() as (fd, tmppath):
if 'content-length' in request.headers: if 'content-length' in request.headers:
fallocate(fd, int(request.headers['content-length'])) fallocate(fd, int(request.headers['content-length']))
chunk_count = 0
dropped_cache = 0
for chunk in iter(lambda: request.body_file.read( for chunk in iter(lambda: request.body_file.read(
self.network_chunk_size), ''): self.network_chunk_size), ''):
upload_size += len(chunk) upload_size += len(chunk)
@ -373,13 +372,11 @@ class ObjectController(object):
while chunk: while chunk:
written = os.write(fd, chunk) written = os.write(fd, chunk)
chunk = chunk[written:] chunk = chunk[written:]
chunk_count += 1
# For large files sync every 512MB (by default) written # For large files sync every 512MB (by default) written
if chunk_count % self.chunks_per_sync == 0: if upload_size - last_sync >= self.bytes_per_sync:
os.fdatasync(fd) os.fdatasync(fd)
drop_buffer_cache(fd, dropped_cache, drop_buffer_cache(fd, last_sync, upload_size - last_sync)
upload_size - dropped_cache) last_sync = upload_size
dropped_cache = upload_size
if 'content-length' in request.headers and \ if 'content-length' in request.headers and \
int(request.headers['content-length']) != upload_size: int(request.headers['content-length']) != upload_size:

2
test/unit/obj/test_server.py
View File

@ -56,7 +56,7 @@ class TestObjectController(unittest.TestCase):
mkdirs(os.path.join(self.testdir, 'sda1', 'tmp')) mkdirs(os.path.join(self.testdir, 'sda1', 'tmp'))
conf = {'devices': self.testdir, 'mount_check': 'false'} conf = {'devices': self.testdir, 'mount_check': 'false'}
self.object_controller = object_server.ObjectController(conf) self.object_controller = object_server.ObjectController(conf)
self.object_controller.chunks_per_sync = 1 self.object_controller.bytes_per_sync = 1
def tearDown(self): def tearDown(self):
""" Tear down for testing swift.object_server.ObjectController """ """ Tear down for testing swift.object_server.ObjectController """