diff --git a/swift/common/middleware/copy.py b/swift/common/middleware/copy.py index b446b1b7b3..a5fc44ca2d 100644 --- a/swift/common/middleware/copy.py +++ b/swift/common/middleware/copy.py @@ -142,7 +142,7 @@ from swift.common.utils import get_logger, \ from swift.common.swob import Request, HTTPPreconditionFailed, \ HTTPRequestEntityTooLarge, HTTPBadRequest from swift.common.http import HTTP_MULTIPLE_CHOICES, HTTP_CREATED, \ - is_success + is_success, HTTP_OK from swift.common.constraints import check_account_format, MAX_FILE_SIZE from swift.common.request_helpers import copy_header_subset, remove_items, \ is_sys_meta, is_sys_or_user_meta @@ -474,7 +474,24 @@ class ServerSideCopyMiddleware(object): # Set data source, content length and etag for the PUT request sink_req.environ['wsgi.input'] = FileLikeIter(source_resp.app_iter) sink_req.content_length = source_resp.content_length - sink_req.etag = source_resp.etag + if (source_resp.status_int == HTTP_OK and + 'X-Static-Large-Object' not in source_resp.headers and + ('X-Object-Manifest' not in source_resp.headers or + req.params.get('multipart-manifest') == 'get')): + # copy source etag so that copied content is verified, unless: + # - not a 200 OK response: source etag may not match the actual + # content, for example with a 206 Partial Content response to a + # ranged request + # - SLO manifest: etag cannot be specified in manifest PUT; SLO + # generates its own etag value which may differ from source + # - SLO: etag in SLO response is not hash of actual content + # - DLO: etag in DLO response is not hash of actual content + sink_req.headers['Etag'] = source_resp.etag + else: + # since we're not copying the source etag, make sure that any + # container update override values are not copied. + remove_items(source_resp.headers, lambda k: k.startswith( + 'X-Object-Sysmeta-Container-Update-Override-')) # We no longer need these headers sink_req.headers.pop('X-Copy-From', None) diff --git a/swift/obj/server.py b/swift/obj/server.py index 99083800eb..7193b73e70 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -447,11 +447,32 @@ class ObjectController(BaseStorageServer): raise HTTPBadRequest("invalid JSON for footer doc") def _check_container_override(self, update_headers, metadata): - for key, val in metadata.items(): - override_prefix = 'x-backend-container-update-override-' - if key.lower().startswith(override_prefix): - override = key.lower().replace(override_prefix, 'x-') - update_headers[override] = val + """ + Applies any overrides to the container update headers. + + Overrides may be in the x-object-sysmeta-container-update- namespace or + the x-backend-container-update-override- namespace. The former is + preferred and is used by proxy middlewares. The latter is historical + but is still used with EC policy PUT requests; for backwards + compatibility the header names used with EC policy requests have not + been changed to the sysmeta namespace - that way the EC PUT path of a + newer proxy will remain compatible with an object server that pre-dates + the introduction of the x-object-sysmeta-container-update- namespace + and vice-versa. + + :param update_headers: a dict of headers used in the container update + :param metadata: a dict that may container override items + """ + # the order of this list is significant: + # x-object-sysmeta-container-update-override-* headers take precedence + # over x-backend-container-update-override-* headers + override_prefixes = ['x-backend-container-update-override-', + 'x-object-sysmeta-container-update-override-'] + for override_prefix in override_prefixes: + for key, val in metadata.items(): + if key.lower().startswith(override_prefix): + override = key.lower().replace(override_prefix, 'x-') + update_headers[override] = val def _preserve_slo_manifest(self, update_metadata, orig_metadata): if 'X-Static-Large-Object' in orig_metadata: diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index af6b9368d7..962cf1bec6 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -1818,6 +1818,11 @@ def trailing_metadata(policy, client_obj_hasher, 'X-Object-Sysmeta-EC-Etag': client_obj_hasher.hexdigest(), 'X-Object-Sysmeta-EC-Content-Length': str(bytes_transferred_from_client), + # older style x-backend-container-update-override-* headers are used + # here (rather than x-object-sysmeta-container-update-override-* + # headers) for backwards compatibility: the request may be to an object + # server that has not yet been upgraded to accept the newer style + # x-object-sysmeta-container-update-override- headers. 'X-Backend-Container-Update-Override-Etag': client_obj_hasher.hexdigest(), 'X-Backend-Container-Update-Override-Size': diff --git a/test/probe/test_object_async_update.py b/test/probe/test_object_async_update.py index b831bbeb72..bab7286424 100755 --- a/test/probe/test_object_async_update.py +++ b/test/probe/test_object_async_update.py @@ -62,7 +62,7 @@ class TestObjectAsyncUpdate(ReplProbeTest): class TestUpdateOverrides(ReplProbeTest): """ Use an internal client to PUT an object to proxy server, - bypassing gatekeeper so that X-Backend- headers can be included. + bypassing gatekeeper so that X-Object-Sysmeta- headers can be included. Verify that the update override headers take effect and override values propagate to the container server. """ @@ -71,10 +71,10 @@ class TestUpdateOverrides(ReplProbeTest): int_client = self.make_internal_client() headers = { 'Content-Type': 'text/plain', - 'X-Backend-Container-Update-Override-Etag': 'override-etag', - 'X-Backend-Container-Update-Override-Content-Type': + 'X-Object-Sysmeta-Container-Update-Override-Etag': 'override-etag', + 'X-Object-Sysmeta-Container-Update-Override-Content-Type': 'override-type', - 'X-Backend-Container-Update-Override-Size': '1999' + 'X-Object-Sysmeta-Container-Update-Override-Size': '1999' } client.put_container(self.url, self.token, 'c1', headers={'X-Storage-Policy': @@ -117,7 +117,8 @@ class TestUpdateOverridesEC(ECProbeTest): # an async update to it kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) content = u'stuff' - client.put_object(self.url, self.token, 'c1', 'o1', contents=content) + client.put_object(self.url, self.token, 'c1', 'o1', contents=content, + content_type='test/ctype') meta = client.head_object(self.url, self.token, 'c1', 'o1') # re-start the container server and assert that it does not yet know @@ -129,11 +130,26 @@ class TestUpdateOverridesEC(ECProbeTest): # Run the object-updaters to be sure updates are done Manager(['object-updater']).once() - # check the re-started container server has update with override values - obj = direct_client.direct_get_container( - cnodes[0], cpart, self.account, 'c1')[1][0] - self.assertEqual(meta['etag'], obj['hash']) - self.assertEqual(len(content), obj['bytes']) + # check the re-started container server got same update as others. + # we cannot assert the actual etag value because it may be encrypted + listing_etags = set() + for cnode in cnodes: + listing = direct_client.direct_get_container( + cnode, cpart, self.account, 'c1')[1] + self.assertEqual(1, len(listing)) + self.assertEqual(len(content), listing[0]['bytes']) + self.assertEqual('test/ctype', listing[0]['content_type']) + listing_etags.add(listing[0]['hash']) + self.assertEqual(1, len(listing_etags)) + + # check that listing meta returned to client is consistent with object + # meta returned to client + hdrs, listing = client.get_container(self.url, self.token, 'c1') + self.assertEqual(1, len(listing)) + self.assertEqual('o1', listing[0]['name']) + self.assertEqual(len(content), listing[0]['bytes']) + self.assertEqual(meta['etag'], listing[0]['hash']) + self.assertEqual('test/ctype', listing[0]['content_type']) def test_update_during_POST_only(self): # verify correct update values when PUT update is missed but then a @@ -147,7 +163,8 @@ class TestUpdateOverridesEC(ECProbeTest): # an async update to it kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) content = u'stuff' - client.put_object(self.url, self.token, 'c1', 'o1', contents=content) + client.put_object(self.url, self.token, 'c1', 'o1', contents=content, + content_type='test/ctype') meta = client.head_object(self.url, self.token, 'c1', 'o1') # re-start the container server and assert that it does not yet know @@ -165,20 +182,39 @@ class TestUpdateOverridesEC(ECProbeTest): int_client.get_object_metadata(self.account, 'c1', 'o1') ['x-object-meta-fruit']) # sanity - # check the re-started container server has update with override values - obj = direct_client.direct_get_container( - cnodes[0], cpart, self.account, 'c1')[1][0] - self.assertEqual(meta['etag'], obj['hash']) - self.assertEqual(len(content), obj['bytes']) + # check the re-started container server got same update as others. + # we cannot assert the actual etag value because it may be encrypted + listing_etags = set() + for cnode in cnodes: + listing = direct_client.direct_get_container( + cnode, cpart, self.account, 'c1')[1] + self.assertEqual(1, len(listing)) + self.assertEqual(len(content), listing[0]['bytes']) + self.assertEqual('test/ctype', listing[0]['content_type']) + listing_etags.add(listing[0]['hash']) + self.assertEqual(1, len(listing_etags)) + + # check that listing meta returned to client is consistent with object + # meta returned to client + hdrs, listing = client.get_container(self.url, self.token, 'c1') + self.assertEqual(1, len(listing)) + self.assertEqual('o1', listing[0]['name']) + self.assertEqual(len(content), listing[0]['bytes']) + self.assertEqual(meta['etag'], listing[0]['hash']) + self.assertEqual('test/ctype', listing[0]['content_type']) # Run the object-updaters to send the async pending from the PUT Manager(['object-updater']).once() # check container listing metadata is still correct - obj = direct_client.direct_get_container( - cnodes[0], cpart, self.account, 'c1')[1][0] - self.assertEqual(meta['etag'], obj['hash']) - self.assertEqual(len(content), obj['bytes']) + for cnode in cnodes: + listing = direct_client.direct_get_container( + cnode, cpart, self.account, 'c1')[1] + self.assertEqual(1, len(listing)) + self.assertEqual(len(content), listing[0]['bytes']) + self.assertEqual('test/ctype', listing[0]['content_type']) + listing_etags.add(listing[0]['hash']) + self.assertEqual(1, len(listing_etags)) def test_async_updates_after_PUT_and_POST(self): # verify correct update values when PUT update and POST updates are @@ -192,7 +228,8 @@ class TestUpdateOverridesEC(ECProbeTest): # we force async updates to it kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) content = u'stuff' - client.put_object(self.url, self.token, 'c1', 'o1', contents=content) + client.put_object(self.url, self.token, 'c1', 'o1', contents=content, + content_type='test/ctype') meta = client.head_object(self.url, self.token, 'c1', 'o1') # use internal client for POST so we can force fast-post mode @@ -213,11 +250,26 @@ class TestUpdateOverridesEC(ECProbeTest): # Run the object-updaters to send the async pendings Manager(['object-updater']).once() - # check container listing metadata is still correct - obj = direct_client.direct_get_container( - cnodes[0], cpart, self.account, 'c1')[1][0] - self.assertEqual(meta['etag'], obj['hash']) - self.assertEqual(len(content), obj['bytes']) + # check the re-started container server got same update as others. + # we cannot assert the actual etag value because it may be encrypted + listing_etags = set() + for cnode in cnodes: + listing = direct_client.direct_get_container( + cnode, cpart, self.account, 'c1')[1] + self.assertEqual(1, len(listing)) + self.assertEqual(len(content), listing[0]['bytes']) + self.assertEqual('test/ctype', listing[0]['content_type']) + listing_etags.add(listing[0]['hash']) + self.assertEqual(1, len(listing_etags)) + + # check that listing meta returned to client is consistent with object + # meta returned to client + hdrs, listing = client.get_container(self.url, self.token, 'c1') + self.assertEqual(1, len(listing)) + self.assertEqual('o1', listing[0]['name']) + self.assertEqual(len(content), listing[0]['bytes']) + self.assertEqual(meta['etag'], listing[0]['hash']) + self.assertEqual('test/ctype', listing[0]['content_type']) if __name__ == '__main__': diff --git a/test/unit/common/middleware/helpers.py b/test/unit/common/middleware/helpers.py index 8b8fff3b3d..c295ee4768 100644 --- a/test/unit/common/middleware/helpers.py +++ b/test/unit/common/middleware/helpers.py @@ -128,6 +128,8 @@ class FakeSwift(object): if "CONTENT_TYPE" in env: self.uploaded[path][0]['Content-Type'] = env["CONTENT_TYPE"] + # note: tests may assume this copy of req_headers is case insensitive + # so we deliberately use a HeaderKeyDict self._calls.append((method, path, HeaderKeyDict(req_headers))) # range requests ought to work, hence conditional_response=True diff --git a/test/unit/common/middleware/test_copy.py b/test/unit/common/middleware/test_copy.py index 254203e630..3f024d4395 100644 --- a/test/unit/common/middleware/test_copy.py +++ b/test/unit/common/middleware/test_copy.py @@ -20,6 +20,7 @@ import shutil import tempfile import unittest from hashlib import md5 +from six.moves import urllib from textwrap import dedent from swift.common import swob @@ -224,9 +225,10 @@ class TestServerSideCopyMiddleware(unittest.TestCase): self.assertEqual('PUT', self.authorized[1].method) self.assertEqual('/v1/a/c/o2', self.authorized[1].path) - def test_static_large_object(self): + def test_static_large_object_manifest(self): self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, - {'X-Static-Large-Object': 'True'}, 'passed') + {'X-Static-Large-Object': 'True', + 'Etag': 'should not be sent'}, 'passed') self.app.register('PUT', '/v1/a/c/o2?multipart-manifest=put', swob.HTTPCreated, {}) req = Request.blank('/v1/a/c/o2?multipart-manifest=get', @@ -236,11 +238,43 @@ class TestServerSideCopyMiddleware(unittest.TestCase): status, headers, body = self.call_ssc(req) self.assertEqual(status, '201 Created') self.assertTrue(('X-Copied-From', 'c/o') in headers) - calls = self.app.calls_with_headers - method, path, req_headers = calls[1] - self.assertEqual('PUT', method) - self.assertEqual('/v1/a/c/o2?multipart-manifest=put', path) + self.assertEqual(2, len(self.app.calls)) + self.assertEqual('GET', self.app.calls[0][0]) + get_path, qs = self.app.calls[0][1].split('?') + params = urllib.parse.parse_qs(qs) + self.assertDictEqual( + {'format': ['raw'], 'multipart-manifest': ['get']}, params) + self.assertEqual(get_path, '/v1/a/c/o') + self.assertEqual(self.app.calls[1], + ('PUT', '/v1/a/c/o2?multipart-manifest=put')) + req_headers = self.app.headers[1] self.assertNotIn('X-Static-Large-Object', req_headers) + self.assertNotIn('Etag', req_headers) + self.assertEqual(len(self.authorized), 2) + self.assertEqual('GET', self.authorized[0].method) + self.assertEqual('/v1/a/c/o', self.authorized[0].path) + self.assertEqual('PUT', self.authorized[1].method) + self.assertEqual('/v1/a/c/o2', self.authorized[1].path) + + def test_static_large_object(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, + {'X-Static-Large-Object': 'True', + 'Etag': 'should not be sent'}, 'passed') + self.app.register('PUT', '/v1/a/c/o2', + swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o2', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': 'c/o'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + self.assertTrue(('X-Copied-From', 'c/o') in headers) + self.assertEqual(self.app.calls, [ + ('GET', '/v1/a/c/o'), + ('PUT', '/v1/a/c/o2')]) + req_headers = self.app.headers[1] + self.assertNotIn('X-Static-Large-Object', req_headers) + self.assertNotIn('Etag', req_headers) self.assertEqual(len(self.authorized), 2) self.assertEqual('GET', self.authorized[0].method) self.assertEqual('/v1/a/c/o', self.authorized[0].path) @@ -587,7 +621,8 @@ class TestServerSideCopyMiddleware(unittest.TestCase): self.assertEqual('/v1/a/c/o', self.authorized[0].path) def test_basic_COPY(self): - self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed') + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, { + 'etag': 'is sent'}, 'passed') self.app.register('PUT', '/v1/a/c/o-copy', swob.HTTPCreated, {}) req = Request.blank( '/v1/a/c/o', method='COPY', @@ -601,6 +636,145 @@ class TestServerSideCopyMiddleware(unittest.TestCase): self.assertEqual('/v1/a/c/o', self.authorized[0].path) self.assertEqual('PUT', self.authorized[1].method) self.assertEqual('/v1/a/c/o-copy', self.authorized[1].path) + self.assertEqual(self.app.calls, [ + ('GET', '/v1/a/c/o'), + ('PUT', '/v1/a/c/o-copy')]) + self.assertIn('etag', self.app.headers[1]) + self.assertEqual(self.app.headers[1]['etag'], 'is sent') + + def test_basic_DLO(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, { + 'x-object-manifest': 'some/path', + 'etag': 'is not sent'}, 'passed') + self.app.register('PUT', '/v1/a/c/o-copy', swob.HTTPCreated, {}) + req = Request.blank( + '/v1/a/c/o', method='COPY', + headers={'Content-Length': 0, + 'Destination': 'c/o-copy'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + self.assertTrue(('X-Copied-From', 'c/o') in headers) + self.assertEqual(self.app.calls, [ + ('GET', '/v1/a/c/o'), + ('PUT', '/v1/a/c/o-copy')]) + self.assertNotIn('x-object-manifest', self.app.headers[1]) + self.assertNotIn('etag', self.app.headers[1]) + + def test_basic_DLO_manifest(self): + self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, { + 'x-object-manifest': 'some/path', + 'etag': 'is sent'}, 'passed') + self.app.register('PUT', '/v1/a/c/o-copy', swob.HTTPCreated, {}) + req = Request.blank( + '/v1/a/c/o?multipart-manifest=get', method='COPY', + headers={'Content-Length': 0, + 'Destination': 'c/o-copy'}) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + self.assertTrue(('X-Copied-From', 'c/o') in headers) + self.assertEqual(2, len(self.app.calls)) + self.assertEqual('GET', self.app.calls[0][0]) + get_path, qs = self.app.calls[0][1].split('?') + params = urllib.parse.parse_qs(qs) + self.assertDictEqual( + {'format': ['raw'], 'multipart-manifest': ['get']}, params) + self.assertEqual(get_path, '/v1/a/c/o') + self.assertEqual(self.app.calls[1], ('PUT', '/v1/a/c/o-copy')) + self.assertIn('x-object-manifest', self.app.headers[1]) + self.assertEqual(self.app.headers[1]['x-object-manifest'], 'some/path') + self.assertIn('etag', self.app.headers[1]) + self.assertEqual(self.app.headers[1]['etag'], 'is sent') + + def test_COPY_source_metadata(self): + source_headers = { + 'x-object-sysmeta-test1': 'copy me', + 'x-object-meta-test2': 'copy me too', + 'x-object-sysmeta-container-update-override-etag': 'etag val', + 'x-object-sysmeta-container-update-override-size': 'size val', + 'x-object-sysmeta-container-update-override-foo': 'bar'} + + get_resp_headers = source_headers.copy() + get_resp_headers['etag'] = 'source etag' + self.app.register( + 'GET', '/v1/a/c/o', swob.HTTPOk, + headers=get_resp_headers, body='passed') + + def verify_headers(expected_headers, unexpected_headers, + actual_headers): + for k, v in actual_headers: + if k.lower() in expected_headers: + expected_val = expected_headers.pop(k.lower()) + self.assertEqual(expected_val, v) + self.assertNotIn(k.lower(), unexpected_headers) + self.assertFalse(expected_headers) + + # use a COPY request + self.app.register('PUT', '/v1/a/c/o-copy0', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o', method='COPY', + headers={'Content-Length': 0, + 'Destination': 'c/o-copy0'}) + status, headers, body = self.call_ssc(req) + self.assertEqual('201 Created', status) + verify_headers(source_headers.copy(), [], headers) + method, path, headers = self.app.calls_with_headers[-1] + self.assertEqual('PUT', method) + self.assertEqual('/v1/a/c/o-copy0', path) + verify_headers(source_headers.copy(), [], headers.items()) + self.assertIn('etag', headers) + self.assertEqual(headers['etag'], 'source etag') + + req = Request.blank('/v1/a/c/o-copy0', method='GET') + status, headers, body = self.call_ssc(req) + self.assertEqual('200 OK', status) + verify_headers(source_headers.copy(), [], headers) + + # use a COPY request with a Range header + self.app.register('PUT', '/v1/a/c/o-copy1', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o', method='COPY', + headers={'Content-Length': 0, + 'Destination': 'c/o-copy1', + 'Range': 'bytes=1-2'}) + status, headers, body = self.call_ssc(req) + expected_headers = source_headers.copy() + unexpected_headers = ( + 'x-object-sysmeta-container-update-override-etag', + 'x-object-sysmeta-container-update-override-size', + 'x-object-sysmeta-container-update-override-foo') + for h in unexpected_headers: + expected_headers.pop(h) + self.assertEqual('201 Created', status) + verify_headers(expected_headers, unexpected_headers, headers) + method, path, headers = self.app.calls_with_headers[-1] + self.assertEqual('PUT', method) + self.assertEqual('/v1/a/c/o-copy1', path) + verify_headers(expected_headers, unexpected_headers, headers.items()) + # etag should not be copied with a Range request + self.assertNotIn('etag', headers) + + req = Request.blank('/v1/a/c/o-copy1', method='GET') + status, headers, body = self.call_ssc(req) + self.assertEqual('200 OK', status) + verify_headers(expected_headers, unexpected_headers, headers) + + # use a PUT with x-copy-from + self.app.register('PUT', '/v1/a/c/o-copy2', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o-copy2', method='PUT', + headers={'Content-Length': 0, + 'X-Copy-From': 'c/o'}) + status, headers, body = self.call_ssc(req) + self.assertEqual('201 Created', status) + verify_headers(source_headers.copy(), [], headers) + method, path, headers = self.app.calls_with_headers[-1] + self.assertEqual('PUT', method) + self.assertEqual('/v1/a/c/o-copy2', path) + verify_headers(source_headers.copy(), [], headers.items()) + self.assertIn('etag', headers) + self.assertEqual(headers['etag'], 'source etag') + + req = Request.blank('/v1/a/c/o-copy2', method='GET') + status, headers, body = self.call_ssc(req) + self.assertEqual('200 OK', status) + verify_headers(source_headers.copy(), [], headers) def test_COPY_no_destination_header(self): req = Request.blank( diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index b85230f395..a40d75c5a2 100755 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -710,6 +710,102 @@ class TestObjectController(unittest.TestCase): self._test_POST_container_updates( POLICIES[1], update_etag='override_etag') + def test_POST_container_updates_precedence(self): + # Verify correct etag and size being sent with container updates for a + # PUT and for a subsequent POST. + ts_iter = make_timestamp_iter() + + def do_test(body, headers, policy): + def mock_container_update(ctlr, op, account, container, obj, req, + headers_out, objdevice, policy): + calls_made.append((headers_out, policy)) + calls_made = [] + ts_put = next(ts_iter) + + # make PUT with given headers and verify correct etag is sent in + # container update + headers.update({ + 'Content-Type': + 'application/octet-stream;swift_bytes=123456789', + 'X-Backend-Storage-Policy-Index': int(policy), + 'X-Object-Sysmeta-Ec-Frag-Index': 2, + 'X-Timestamp': ts_put.internal, + 'Content-Length': len(body)}) + + req = Request.blank('/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'PUT'}, + headers=headers, body=body) + + with mock.patch( + 'swift.obj.server.ObjectController.container_update', + mock_container_update): + resp = req.get_response(self.object_controller) + + self.assertEqual(resp.status_int, 201) + self.assertEqual(1, len(calls_made)) + expected_headers = HeaderKeyDict({ + 'x-size': '4', + 'x-content-type': + 'application/octet-stream;swift_bytes=123456789', + 'x-timestamp': ts_put.internal, + 'x-etag': 'expected'}) + self.assertDictEqual(expected_headers, calls_made[0][0]) + self.assertEqual(policy, calls_made[0][1]) + + # make a POST and verify container update has the same etag + calls_made = [] + ts_post = next(ts_iter) + req = Request.blank( + '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': ts_post.internal, + 'X-Backend-Storage-Policy-Index': int(policy)}) + + with mock.patch( + 'swift.obj.server.ObjectController.container_update', + mock_container_update): + resp = req.get_response(self.object_controller) + + self.assertEqual(resp.status_int, 202) + self.assertEqual(1, len(calls_made)) + expected_headers.update({ + 'x-content-type-timestamp': ts_put.internal, + 'x-meta-timestamp': ts_post.internal}) + self.assertDictEqual(expected_headers, calls_made[0][0]) + self.assertEqual(policy, calls_made[0][1]) + + # sanity check - EC headers are ok + headers = { + 'X-Backend-Container-Update-Override-Etag': 'expected', + 'X-Backend-Container-Update-Override-Size': '4', + 'X-Object-Sysmeta-Ec-Etag': 'expected', + 'X-Object-Sysmeta-Ec-Content-Length': '4'} + do_test('test ec frag longer than 4', headers, POLICIES[1]) + + # middleware overrides take precedence over EC/older overrides + headers = { + 'X-Backend-Container-Update-Override-Etag': 'unexpected', + 'X-Backend-Container-Update-Override-Size': '3', + 'X-Object-Sysmeta-Ec-Etag': 'unexpected', + 'X-Object-Sysmeta-Ec-Content-Length': '3', + 'X-Object-Sysmeta-Container-Update-Override-Etag': 'expected', + 'X-Object-Sysmeta-Container-Update-Override-Size': '4'} + do_test('test ec frag longer than 4', headers, POLICIES[1]) + + # overrides with replication policy + headers = { + 'X-Object-Sysmeta-Container-Update-Override-Etag': 'expected', + 'X-Object-Sysmeta-Container-Update-Override-Size': '4'} + do_test('longer than 4', headers, POLICIES[0]) + + # middleware overrides take precedence over EC/older overrides with + # replication policy + headers = { + 'X-Backend-Container-Update-Override-Etag': 'unexpected', + 'X-Backend-Container-Update-Override-Size': '3', + 'X-Object-Sysmeta-Container-Update-Override-Etag': 'expected', + 'X-Object-Sysmeta-Container-Update-Override-Size': '4'} + do_test('longer than 4', headers, POLICIES[0]) + def _test_PUT_then_POST_async_pendings(self, policy, update_etag=None): # Test that PUT and POST requests result in distinct async pending # files when sync container update fails. @@ -4310,47 +4406,75 @@ class TestObjectController(unittest.TestCase): 'x-trans-id': '123', 'referer': 'PUT http://localhost/sda1/0/a/c/o'})) - def test_container_update_overrides(self): - container_updates = [] + def test_PUT_container_update_overrides(self): + ts_iter = make_timestamp_iter() - def capture_updates(ip, port, method, path, headers, *args, **kwargs): - container_updates.append((ip, port, method, path, headers)) + def do_test(override_headers): + container_updates = [] - headers = { - 'X-Timestamp': 1, - 'X-Trans-Id': '123', - 'X-Container-Host': 'chost:cport', - 'X-Container-Partition': 'cpartition', - 'X-Container-Device': 'cdevice', - 'Content-Type': 'text/plain', + def capture_updates( + ip, port, method, path, headers, *args, **kwargs): + container_updates.append((ip, port, method, path, headers)) + + ts_put = next(ts_iter) + headers = { + 'X-Timestamp': ts_put.internal, + 'X-Trans-Id': '123', + 'X-Container-Host': 'chost:cport', + 'X-Container-Partition': 'cpartition', + 'X-Container-Device': 'cdevice', + 'Content-Type': 'text/plain', + } + headers.update(override_headers) + req = Request.blank('/sda1/0/a/c/o', method='PUT', + headers=headers, body='') + with mocked_http_conn( + 200, give_connect=capture_updates) as fake_conn: + with fake_spawn(): + resp = req.get_response(self.object_controller) + self.assertRaises(StopIteration, fake_conn.code_iter.next) + self.assertEqual(resp.status_int, 201) + self.assertEqual(len(container_updates), 1) + ip, port, method, path, headers = container_updates[0] + self.assertEqual(ip, 'chost') + self.assertEqual(port, 'cport') + self.assertEqual(method, 'PUT') + self.assertEqual(path, '/cdevice/cpartition/a/c/o') + self.assertEqual(headers, HeaderKeyDict({ + 'user-agent': 'object-server %s' % os.getpid(), + 'x-size': '0', + 'x-etag': 'override_etag', + 'x-content-type': 'override_val', + 'x-timestamp': ts_put.internal, + 'X-Backend-Storage-Policy-Index': '0', # default + 'x-trans-id': '123', + 'referer': 'PUT http://localhost/sda1/0/a/c/o', + 'x-foo': 'bar'})) + + # EC policy override headers + do_test({ 'X-Backend-Container-Update-Override-Etag': 'override_etag', 'X-Backend-Container-Update-Override-Content-Type': 'override_val', 'X-Backend-Container-Update-Override-Foo': 'bar', - 'X-Backend-Container-Ignored': 'ignored' - } - req = Request.blank('/sda1/0/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers=headers, body='') - with mocked_http_conn(200, give_connect=capture_updates) as fake_conn: - with fake_spawn(): - resp = req.get_response(self.object_controller) - self.assertRaises(StopIteration, fake_conn.code_iter.next) - self.assertEqual(resp.status_int, 201) - self.assertEqual(len(container_updates), 1) - ip, port, method, path, headers = container_updates[0] - self.assertEqual(ip, 'chost') - self.assertEqual(port, 'cport') - self.assertEqual(method, 'PUT') - self.assertEqual(path, '/cdevice/cpartition/a/c/o') - self.assertEqual(headers, HeaderKeyDict({ - 'user-agent': 'object-server %s' % os.getpid(), - 'x-size': '0', - 'x-etag': 'override_etag', - 'x-content-type': 'override_val', - 'x-timestamp': utils.Timestamp(1).internal, - 'X-Backend-Storage-Policy-Index': '0', # default when not given - 'x-trans-id': '123', - 'referer': 'PUT http://localhost/sda1/0/a/c/o', - 'x-foo': 'bar'})) + 'X-Backend-Container-Ignored': 'ignored'}) + + # middleware override headers + do_test({ + 'X-Object-Sysmeta-Container-Update-Override-Etag': 'override_etag', + 'X-Object-Sysmeta-Container-Update-Override-Content-Type': + 'override_val', + 'X-Object-Sysmeta-Container-Update-Override-Foo': 'bar', + 'X-Object-Sysmeta-Ignored': 'ignored'}) + + # middleware override headers take precedence over EC policy headers + do_test({ + 'X-Object-Sysmeta-Container-Update-Override-Etag': 'override_etag', + 'X-Object-Sysmeta-Container-Update-Override-Content-Type': + 'override_val', + 'X-Object-Sysmeta-Container-Update-Override-Foo': 'bar', + 'X-Backend-Container-Update-Override-Etag': 'ignored', + 'X-Backend-Container-Update-Override-Content-Type': 'ignored', + 'X-Backend-Container-Update-Override-Foo': 'ignored'}) def test_container_update_async(self): policy = random.choice(list(POLICIES))