openstack/swift
Code Issues Proposed changes

Merge "Push fallocate() down into mkstemp(); use known size"

Browse Source
This commit is contained in:
Jenkins 2013-04-29 04:44:26 +00:00 committed by Gerrit Code Review
commit 58259df8df
4 changed files with 92 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
swift/common/exceptions.py
View File

@ -50,6 +50,10 @@ class DiskFileNotExist(SwiftException):
pass pass
class DiskFileNoSpace(SwiftException):
pass
class PathNotDir(OSError): class PathNotDir(OSError):
pass pass

146
swift/obj/server.py
View File

@ -38,7 +38,7 @@ from swift.common.bufferedhttp import http_connect
from swift.common.constraints import check_object_creation, check_mount, \ from swift.common.constraints import check_object_creation, check_mount, \
check_float, check_utf8 check_float, check_utf8
from swift.common.exceptions import ConnectionTimeout, DiskFileError, \ from swift.common.exceptions import ConnectionTimeout, DiskFileError, \
DiskFileNotExist, DiskFileCollision DiskFileNotExist, DiskFileCollision, DiskFileNoSpace
from swift.obj.replicator import tpool_reraise, invalidate_hash, \ from swift.obj.replicator import tpool_reraise, invalidate_hash, \
quarantine_renamer, get_hashes quarantine_renamer, get_hashes
from swift.common.http import is_success from swift.common.http import is_success
@ -284,12 +284,22 @@ class DiskFile(object):
int(self.metadata['X-Delete-At']) <= time.time()) int(self.metadata['X-Delete-At']) <= time.time())
@contextmanager @contextmanager
def mkstemp(self): def mkstemp(self, size=None):
"""Contextmanager to make a temporary file.""" """
Contextmanager to make a temporary file.
:param size: optional initial size of file to allocate on disk
:raises DiskFileNoSpace: if a size is specified and fallocate fails
"""
if not os.path.exists(self.tmpdir): if not os.path.exists(self.tmpdir):
mkdirs(self.tmpdir) mkdirs(self.tmpdir)
fd, self.tmppath = mkstemp(dir=self.tmpdir) fd, self.tmppath = mkstemp(dir=self.tmpdir)
try: try:
if size is not None and size > 0:
try:
fallocate(fd, size)
except OSError:
raise DiskFileNoSpace()
yield fd yield fd
finally: finally:
try: try:
@ -302,13 +312,14 @@ class DiskFile(object):
except OSError: except OSError:
pass pass
def put(self, fd, metadata, extension='.data'): def put(self, fd, fsize, metadata, extension='.data'):
""" """
Finalize writing the file on disk, and renames it from the temp file to Finalize writing the file on disk, and renames it from the temp file to
the real location. This should be called after the data has been the real location. This should be called after the data has been
written to the temp file. written to the temp file.
:param fd: file descriptor of the temp file :param fd: file descriptor of the temp file
:param fsize: final on-disk size of the created file
:param metadata: dictionary of metadata to be written :param metadata: dictionary of metadata to be written
:param extension: extension to be used when making the file :param extension: extension to be used when making the file
""" """
@ -322,11 +333,10 @@ class DiskFile(object):
# redundant work the drop cache code will perform on the pages (now # redundant work the drop cache code will perform on the pages (now
# that after fsync the pages will be all clean). # that after fsync the pages will be all clean).
tpool.execute(fsync, fd) tpool.execute(fsync, fd)
if 'Content-Length' in metadata: # From the Department of the Redundancy Department, make sure we
# From the Department of the Redundancy Department, make sure we # call drop_cache() after fsync() to avoid redundant work (pages
# call drop_cache() after fsync() to avoid redundant work (pages # all clean).
# all clean). self.drop_cache(fd, 0, fsize)
self.drop_cache(fd, 0, int(metadata['Content-Length']))
invalidate_hash(os.path.dirname(self.datadir)) invalidate_hash(os.path.dirname(self.datadir))
# After the rename completes, this object will be available for other # After the rename completes, this object will be available for other
# requests to reference. # requests to reference.
@ -343,7 +353,7 @@ class DiskFile(object):
""" """
extension = '.ts' if tombstone else '.meta' extension = '.ts' if tombstone else '.meta'
with self.mkstemp() as fd: with self.mkstemp() as fd:
self.put(fd, metadata, extension=extension) self.put(fd, 0, metadata, extension=extension)
def unlinkold(self, timestamp): def unlinkold(self, timestamp):
""" """
@ -660,68 +670,70 @@ class ObjectController(object):
orig_timestamp = file.metadata.get('X-Timestamp') orig_timestamp = file.metadata.get('X-Timestamp')
upload_expiration = time.time() + self.max_upload_time upload_expiration = time.time() + self.max_upload_time
etag = md5() etag = md5()
fsize = request.headers.get('content-length', None)
if fsize is not None:
fsize = int(fsize)
upload_size = 0 upload_size = 0
last_sync = 0 last_sync = 0
elapsed_time = 0 elapsed_time = 0
with file.mkstemp() as fd: try:
try: with file.mkstemp(size=fsize) as fd:
fallocate(fd, int(request.headers.get('content-length', 0))) reader = request.environ['wsgi.input'].read
except OSError: for chunk in iter(lambda: reader(self.network_chunk_size), ''):
return HTTPInsufficientStorage(drive=device, request=request) start_time = time.time()
reader = request.environ['wsgi.input'].read upload_size += len(chunk)
for chunk in iter(lambda: reader(self.network_chunk_size), ''): if time.time() > upload_expiration:
start_time = time.time() self.logger.increment('PUT.timeouts')
upload_size += len(chunk) return HTTPRequestTimeout(request=request)
if time.time() > upload_expiration: etag.update(chunk)
self.logger.increment('PUT.timeouts') while chunk:
return HTTPRequestTimeout(request=request) written = os.write(fd, chunk)
etag.update(chunk) chunk = chunk[written:]
while chunk: # For large files sync every 512MB (by default) written
written = os.write(fd, chunk) if upload_size - last_sync >= self.bytes_per_sync:
chunk = chunk[written:] tpool.execute(fdatasync, fd)
# For large files sync every 512MB (by default) written drop_buffer_cache(fd, last_sync,
if upload_size - last_sync >= self.bytes_per_sync: upload_size - last_sync)
tpool.execute(fdatasync, fd) last_sync = upload_size
drop_buffer_cache(fd, last_sync, upload_size - last_sync) sleep()
last_sync = upload_size elapsed_time += time.time() - start_time
sleep()
elapsed_time += time.time() - start_time
if upload_size: if upload_size:
self.logger.transfer_rate( self.logger.transfer_rate(
'PUT.' + device + '.timing', elapsed_time, upload_size) 'PUT.' + device + '.timing', elapsed_time, upload_size)
if 'content-length' in request.headers and \ if fsize is not None and fsize != upload_size:
int(request.headers['content-length']) != upload_size: return HTTPClientDisconnect(request=request)
return HTTPClientDisconnect(request=request) etag = etag.hexdigest()
etag = etag.hexdigest() if 'etag' in request.headers and \
if 'etag' in request.headers and \ request.headers['etag'].lower() != etag:
request.headers['etag'].lower() != etag: return HTTPUnprocessableEntity(request=request)
return HTTPUnprocessableEntity(request=request) metadata = {
metadata = { 'X-Timestamp': request.headers['x-timestamp'],
'X-Timestamp': request.headers['x-timestamp'], 'Content-Type': request.headers['content-type'],
'Content-Type': request.headers['content-type'], 'ETag': etag,
'ETag': etag, 'Content-Length': str(upload_size),
'Content-Length': str(upload_size), }
} metadata.update(val for val in request.headers.iteritems()
metadata.update(val for val in request.headers.iteritems() if val[0].lower().startswith('x-object-meta-')
if val[0].lower().startswith('x-object-meta-') and and len(val[0]) > 14)
len(val[0]) > 14) for header_key in self.allowed_headers:
for header_key in self.allowed_headers: if header_key in request.headers:
if header_key in request.headers: header_caps = header_key.title()
header_caps = header_key.title() metadata[header_caps] = request.headers[header_key]
metadata[header_caps] = request.headers[header_key] old_delete_at = int(file.metadata.get('X-Delete-At') or 0)
old_delete_at = int(file.metadata.get('X-Delete-At') or 0) if old_delete_at != new_delete_at:
if old_delete_at != new_delete_at: if new_delete_at:
if new_delete_at: self.delete_at_update(
self.delete_at_update( 'PUT', new_delete_at, account, container, obj,
'PUT', new_delete_at, account, container, obj, request.headers, device)
request.headers, device) if old_delete_at:
if old_delete_at: self.delete_at_update(
self.delete_at_update( 'DELETE', old_delete_at, account, container, obj,
'DELETE', old_delete_at, account, container, obj, request.headers, device)
request.headers, device) file.put(fd, upload_size, metadata)
file.put(fd, metadata) except DiskFileNoSpace:
return HTTPInsufficientStorage(drive=device, request=request)
file.unlinkold(metadata['X-Timestamp']) file.unlinkold(metadata['X-Timestamp'])
if not orig_timestamp or \ if not orig_timestamp or \
orig_timestamp < request.headers['x-timestamp']: orig_timestamp < request.headers['x-timestamp']:

16
test/unit/obj/test_auditor.py
View File

@ -72,7 +72,7 @@ class TestAuditor(unittest.TestCase):
'X-Timestamp': timestamp, 'X-Timestamp': timestamp,
'Content-Length': str(os.fstat(fd).st_size), 'Content-Length': str(os.fstat(fd).st_size),
} }
self.disk_file.put(fd, metadata) self.disk_file.put(fd, 1024, metadata)
pre_quarantines = self.auditor.quarantines pre_quarantines = self.auditor.quarantines
self.auditor.object_audit( self.auditor.object_audit(
@ -100,7 +100,7 @@ class TestAuditor(unittest.TestCase):
'X-Timestamp': timestamp, 'X-Timestamp': timestamp,
'Content-Length': str(os.fstat(fd).st_size), 'Content-Length': str(os.fstat(fd).st_size),
} }
self.disk_file.put(fd, metadata) self.disk_file.put(fd, 1024, metadata)
pre_quarantines = self.auditor.quarantines pre_quarantines = self.auditor.quarantines
# remake so it will have metadata # remake so it will have metadata
self.disk_file = DiskFile(self.devices, 'sda', '0', 'a', 'c', 'o', self.disk_file = DiskFile(self.devices, 'sda', '0', 'a', 'c', 'o',
@ -161,7 +161,7 @@ class TestAuditor(unittest.TestCase):
'X-Timestamp': timestamp, 'X-Timestamp': timestamp,
'Content-Length': str(os.fstat(fd).st_size), 'Content-Length': str(os.fstat(fd).st_size),
} }
self.disk_file.put(fd, metadata) self.disk_file.put(fd, 1024, metadata)
self.disk_file.close() self.disk_file.close()
self.auditor.audit_all_objects() self.auditor.audit_all_objects()
self.assertEquals(self.auditor.quarantines, pre_quarantines) self.assertEquals(self.auditor.quarantines, pre_quarantines)
@ -181,7 +181,7 @@ class TestAuditor(unittest.TestCase):
'X-Timestamp': timestamp, 'X-Timestamp': timestamp,
'Content-Length': str(os.fstat(fd).st_size), 'Content-Length': str(os.fstat(fd).st_size),
} }
self.disk_file.put(fd, metadata) self.disk_file.put(fd, 1024, metadata)
self.disk_file.close() self.disk_file.close()
os.write(fd, 'extra_data') os.write(fd, 'extra_data')
self.auditor.audit_all_objects() self.auditor.audit_all_objects()
@ -202,7 +202,7 @@ class TestAuditor(unittest.TestCase):
'X-Timestamp': timestamp, 'X-Timestamp': timestamp,
'Content-Length': str(os.fstat(fd).st_size), 'Content-Length': str(os.fstat(fd).st_size),
} }
self.disk_file.put(fd, metadata) self.disk_file.put(fd, 10, metadata)
self.disk_file.close() self.disk_file.close()
self.auditor.audit_all_objects() self.auditor.audit_all_objects()
self.disk_file = DiskFile(self.devices, 'sdb', '0', 'a', 'c', self.disk_file = DiskFile(self.devices, 'sdb', '0', 'a', 'c',
@ -218,7 +218,7 @@ class TestAuditor(unittest.TestCase):
'X-Timestamp': timestamp, 'X-Timestamp': timestamp,
'Content-Length': str(os.fstat(fd).st_size), 'Content-Length': str(os.fstat(fd).st_size),
} }
self.disk_file.put(fd, metadata) self.disk_file.put(fd, 10, metadata)
self.disk_file.close() self.disk_file.close()
os.write(fd, 'extra_data') os.write(fd, 'extra_data')
self.auditor.audit_all_objects() self.auditor.audit_all_objects()
@ -238,7 +238,7 @@ class TestAuditor(unittest.TestCase):
'X-Timestamp': str(normalize_timestamp(time.time())), 'X-Timestamp': str(normalize_timestamp(time.time())),
'Content-Length': str(os.fstat(fd).st_size), 'Content-Length': str(os.fstat(fd).st_size),
} }
self.disk_file.put(fd, metadata) self.disk_file.put(fd, 1024, metadata)
etag = md5() etag = md5()
etag.update('1' + '0' * 1023) etag.update('1' + '0' * 1023)
etag = etag.hexdigest() etag = etag.hexdigest()
@ -275,7 +275,7 @@ class TestAuditor(unittest.TestCase):
'X-Timestamp': str(normalize_timestamp(time.time())), 'X-Timestamp': str(normalize_timestamp(time.time())),
'Content-Length': 10, 'Content-Length': 10,
} }
self.disk_file.put(fd, metadata) self.disk_file.put(fd, 10, metadata)
etag = md5() etag = md5()
etag = etag.hexdigest() etag = etag.hexdigest()
metadata['ETag'] = etag metadata['ETag'] = etag

2
test/unit/obj/test_server.py
View File

@ -224,7 +224,7 @@ class TestDiskFile(unittest.TestCase):
'X-Timestamp': timestamp, 'X-Timestamp': timestamp,
'Content-Length': str(os.fstat(fd).st_size), 'Content-Length': str(os.fstat(fd).st_size),
} }
df.put(fd, metadata, extension=extension) df.put(fd, fsize, metadata, extension=extension)
if invalid_type == 'ETag': if invalid_type == 'ETag':
etag = md5() etag = md5()
etag.update('1' + '0' * (fsize - 1)) etag.update('1' + '0' * (fsize - 1))