diff --git a/swift/obj/mem_diskfile.py b/swift/obj/mem_diskfile.py index fe1dc5e496..c233f57b09 100644 --- a/swift/obj/mem_diskfile.py +++ b/swift/obj/mem_diskfile.py @@ -25,7 +25,9 @@ from six import moves from swift.common.utils import Timestamp from swift.common.exceptions import DiskFileQuarantined, DiskFileNotExist, \ DiskFileCollision, DiskFileDeleted, DiskFileNotOpen +from swift.common.request_helpers import is_sys_meta from swift.common.swob import multi_range_iterator +from swift.obj.diskfile import DATAFILE_SYSTEM_META class InMemoryFileSystem(object): @@ -99,7 +101,6 @@ class DiskFileWriter(object): with the `StringIO` object. :param metadata: dictionary of metadata to be written - :param extension: extension to be used when making the file """ metadata['name'] = self._name self._filesystem.put_object(self._name, self._fp, metadata) @@ -209,7 +210,7 @@ class DiskFileReader(object): if self._bytes_read != self._obj_size: self._quarantine( "Bytes read: %s, does not match metadata: %s" % ( - self.bytes_read, self._obj_size)) + self._bytes_read, self._obj_size)) elif self._iter_etag and \ self._etag != self._iter_etag.hexdigest(): self._quarantine( @@ -239,14 +240,10 @@ class DiskFile(object): Manage object files in-memory. - :param mgr: DiskFileManager - :param device_path: path to the target device or drive - :param threadpool: thread pool to use for blocking operations - :param partition: partition on the device in which the object lives + :param fs: an instance of InMemoryFileSystem :param account: account name for the object :param container: container name for the object :param obj: object name for the object - :param keep_cache: caller's preference for keeping data read in the cache """ def __init__(self, fs, account, container, obj): @@ -283,6 +280,19 @@ class DiskFile(object): if self._fp is not None: self._fp = None + def _quarantine(self, name, msg): + """ + Quarantine a file; responsible for incrementing the associated logger's + count of quarantines. + + :param name: name of object to quarantine + :param msg: reason for quarantining to be included in the exception + :returns: DiskFileQuarantined exception object + """ + # for this implementation we simply delete the bad object + self._filesystem.del_object(name) + return DiskFileQuarantined(msg) + def _verify_data_file(self, fp): """ Verify the metadata's name value matches what we think the object is @@ -396,9 +406,18 @@ class DiskFile(object): """ Write a block of metadata to an object. """ - cur_fp = self._filesystem.get(self._name) - if cur_fp is not None: - self._filesystem[self._name] = (cur_fp, metadata) + data, cur_mdata = self._filesystem.get_object(self._name) + if data is not None: + # The object exists. Update the new metadata with the object's + # immutable metadata (e.g. name, size, etag, sysmeta) and store it + # with the object data. + immutable_metadata = dict( + [(key, val) for key, val in cur_mdata.items() + if key.lower() in DATAFILE_SYSTEM_META + or is_sys_meta('object', key)]) + metadata.update(immutable_metadata) + metadata['name'] = self._name + self._filesystem.put_object(self._name, data, metadata) def delete(self, timestamp): """ diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 0133056eeb..dc9a5a8b3d 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -2898,8 +2898,50 @@ class TestObjectController(unittest.TestCase): self.assertEqual(headers[:len(exp)], exp) @unpatch_policies - def test_PUT_last_modified(self): + def test_PUT_POST_last_modified(self): prolis = _test_sockets[0] + prosrv = _test_servers[0] + + def _do_HEAD(): + # do a HEAD to get reported last modified time + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('HEAD /v1/a/c/o.last_modified HTTP/1.1\r\n' + 'Host: localhost\r\nConnection: close\r\n' + 'X-Storage-Token: t\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 200' + self.assertEqual(headers[:len(exp)], exp) + last_modified_head = [line for line in headers.split('\r\n') + if lm_hdr in line][0][len(lm_hdr):] + return last_modified_head + + def _do_conditional_GET_checks(last_modified_time): + # check If-(Un)Modified-Since GETs + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a/c/o.last_modified HTTP/1.1\r\n' + 'Host: localhost\r\nConnection: close\r\n' + 'If-Modified-Since: %s\r\n' + 'X-Storage-Token: t\r\n\r\n' % last_modified_time) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 304' + self.assertEqual(headers[:len(exp)], exp) + + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a/c/o.last_modified HTTP/1.1\r\n' + 'Host: localhost\r\nConnection: close\r\n' + 'If-Unmodified-Since: %s\r\n' + 'X-Storage-Token: t\r\n\r\n' % last_modified_time) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 200' + self.assertEqual(headers[:len(exp)], exp) + + # PUT the object sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a/c/o.last_modified HTTP/1.1\r\n' @@ -2913,40 +2955,50 @@ class TestObjectController(unittest.TestCase): last_modified_put = [line for line in headers.split('\r\n') if lm_hdr in line][0][len(lm_hdr):] - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('HEAD /v1/a/c/o.last_modified HTTP/1.1\r\n' - 'Host: localhost\r\nConnection: close\r\n' - 'X-Storage-Token: t\r\n\r\n') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 200' - self.assertEqual(headers[:len(exp)], exp) - last_modified_head = [line for line in headers.split('\r\n') - if lm_hdr in line][0][len(lm_hdr):] + + last_modified_head = _do_HEAD() self.assertEqual(last_modified_put, last_modified_head) - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('GET /v1/a/c/o.last_modified HTTP/1.1\r\n' - 'Host: localhost\r\nConnection: close\r\n' - 'If-Modified-Since: %s\r\n' - 'X-Storage-Token: t\r\n\r\n' % last_modified_put) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 304' - self.assertEqual(headers[:len(exp)], exp) + _do_conditional_GET_checks(last_modified_put) + # now POST to the object using default object_post_as_copy setting + orig_post_as_copy = prosrv.object_post_as_copy sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() - fd.write('GET /v1/a/c/o.last_modified HTTP/1.1\r\n' + fd.write('POST /v1/a/c/o.last_modified HTTP/1.1\r\n' 'Host: localhost\r\nConnection: close\r\n' - 'If-Unmodified-Since: %s\r\n' - 'X-Storage-Token: t\r\n\r\n' % last_modified_put) + 'X-Storage-Token: t\r\nContent-Length: 0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 200' + exp = 'HTTP/1.1 202' self.assertEqual(headers[:len(exp)], exp) + for line in headers.split('\r\n'): + self.assertFalse(line.startswith(lm_hdr)) + + # last modified time will have changed due to POST + last_modified_head = _do_HEAD() + _do_conditional_GET_checks(last_modified_head) + + # now POST using non-default object_post_as_copy setting + try: + prosrv.object_post_as_copy = not orig_post_as_copy + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('POST /v1/a/c/o.last_modified HTTP/1.1\r\n' + 'Host: localhost\r\nConnection: close\r\n' + 'X-Storage-Token: t\r\nContent-Length: 0\r\n\r\n') + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 202' + self.assertEqual(headers[:len(exp)], exp) + for line in headers.split('\r\n'): + self.assertFalse(line.startswith(lm_hdr)) + finally: + prosrv.object_post_as_copy = orig_post_as_copy + + # last modified time will have changed due to POST + last_modified_head = _do_HEAD() + _do_conditional_GET_checks(last_modified_head) def test_PUT_auto_content_type(self): with save_globals():