diff --git a/swift/common/middleware/copy.py b/swift/common/middleware/copy.py index 9ef44a2243..4d6c52d77b 100644 --- a/swift/common/middleware/copy.py +++ b/swift/common/middleware/copy.py @@ -455,10 +455,33 @@ class ServerSideCopyMiddleware(object): close_if_possible(source_resp.app_iter) return source_resp(source_resp.environ, start_response) - # Create a new Request object based on the original req instance. - # This will preserve env and headers. - sink_req = Request.blank(req.path_info, - environ=req.environ, headers=req.headers) + # Create a new Request object based on the original request instance. + # This will preserve original request environ including headers. + sink_req = Request.blank(req.path_info, environ=req.environ) + + def is_object_sysmeta(k): + return is_sys_meta('object', k) + + if 'swift.post_as_copy' in sink_req.environ: + # Post-as-copy: ignore new sysmeta, copy existing sysmeta + remove_items(sink_req.headers, is_object_sysmeta) + copy_header_subset(source_resp, sink_req, is_object_sysmeta) + elif config_true_value(req.headers.get('x-fresh-metadata', 'false')): + # x-fresh-metadata only applies to copy, not post-as-copy: ignore + # existing user metadata, update existing sysmeta with new + copy_header_subset(source_resp, sink_req, is_object_sysmeta) + copy_header_subset(req, sink_req, is_object_sysmeta) + else: + # First copy existing sysmeta, user meta and other headers from the + # source to the sink, apart from headers that are conditionally + # copied below and timestamps. + exclude_headers = ('x-static-large-object', 'x-object-manifest', + 'etag', 'content-type', 'x-timestamp', + 'x-backend-timestamp') + copy_header_subset(source_resp, sink_req, + lambda k: k.lower() not in exclude_headers) + # now update with original req headers + sink_req.headers.update(req.headers) params = sink_req.params if params.get('multipart-manifest') == 'get': @@ -489,32 +512,19 @@ class ServerSideCopyMiddleware(object): else: # since we're not copying the source etag, make sure that any # container update override values are not copied. - remove_items(source_resp.headers, lambda k: k.startswith( + remove_items(sink_req.headers, lambda k: k.startswith( 'X-Object-Sysmeta-Container-Update-Override-')) # We no longer need these headers sink_req.headers.pop('X-Copy-From', None) sink_req.headers.pop('X-Copy-From-Account', None) + # If the copy request does not explicitly override content-type, # use the one present in the source object. if not req.headers.get('content-type'): sink_req.headers['Content-Type'] = \ source_resp.headers['Content-Type'] - fresh_meta_flag = config_true_value( - sink_req.headers.get('x-fresh-metadata', 'false')) - - if fresh_meta_flag or 'swift.post_as_copy' in sink_req.environ: - # Post-as-copy: ignore new sysmeta, copy existing sysmeta - condition = lambda k: is_sys_meta('object', k) - remove_items(sink_req.headers, condition) - copy_header_subset(source_resp, sink_req, condition) - else: - # Copy/update existing sysmeta, transient-sysmeta and user meta - _copy_headers(source_resp.headers, sink_req.headers) - # Copy/update new metadata provided in request if any - _copy_headers(req.headers, sink_req.headers) - # Create response headers for PUT response resp_headers = self._create_response_headers(source_path, source_resp, sink_req) diff --git a/test/functional/tests.py b/test/functional/tests.py index 156094ac4b..29194964d1 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -1186,11 +1186,24 @@ class TestFile(Base): file_item = self.env.container.file(source_filename) metadata = {} - for i in range(1): - metadata[Utils.create_ascii_name()] = Utils.create_name() + metadata[Utils.create_ascii_name()] = Utils.create_name() + put_headers = {'Content-Type': 'application/test', + 'Content-Encoding': 'gzip', + 'Content-Disposition': 'attachment; filename=myfile'} + file_item.metadata = metadata + data = file_item.write_random(hdrs=put_headers) - data = file_item.write_random() - file_item.sync_metadata(metadata) + # the allowed headers are configurable in object server, so we cannot + # assert that content-encoding and content-disposition get *copied* + # unless they were successfully set on the original PUT, so populate + # expected_headers by making a HEAD on the original object + file_item.initialize() + self.assertEqual('application/test', file_item.content_type) + resp_headers = dict(file_item.conn.response.getheaders()) + expected_headers = {} + for k, v in put_headers.items(): + if k.lower() in resp_headers: + expected_headers[k] = v dest_cont = self.env.account.container(Utils.create_name()) self.assertTrue(dest_cont.create()) @@ -1201,16 +1214,71 @@ class TestFile(Base): for prefix in ('', '/'): dest_filename = Utils.create_name() - file_item = self.env.container.file(source_filename) - file_item.copy('%s%s' % (prefix, cont), dest_filename) + extra_hdrs = {'X-Object-Meta-Extra': 'fresh'} + self.assertTrue(file_item.copy( + '%s%s' % (prefix, cont), dest_filename, hdrs=extra_hdrs)) self.assertIn(dest_filename, cont.files()) - file_item = cont.file(dest_filename) + file_copy = cont.file(dest_filename) - self.assertEqual(data, file_item.read()) - self.assertTrue(file_item.initialize()) - self.assertEqual(metadata, file_item.metadata) + self.assertEqual(data, file_copy.read()) + self.assertTrue(file_copy.initialize()) + expected_metadata = dict(metadata) + # new metadata should be merged with existing + expected_metadata['extra'] = 'fresh' + self.assertDictEqual(expected_metadata, file_copy.metadata) + resp_headers = dict(file_copy.conn.response.getheaders()) + for k, v in expected_headers.items(): + self.assertIn(k.lower(), resp_headers) + self.assertEqual(v, resp_headers[k.lower()]) + + # repeat copy with updated content-type, content-encoding and + # content-disposition, which should get updated + extra_hdrs = { + 'X-Object-Meta-Extra': 'fresher', + 'Content-Type': 'application/test-changed', + 'Content-Encoding': 'not_gzip', + 'Content-Disposition': 'attachment; filename=notmyfile'} + self.assertTrue(file_item.copy( + '%s%s' % (prefix, cont), dest_filename, hdrs=extra_hdrs)) + + self.assertIn(dest_filename, cont.files()) + + file_copy = cont.file(dest_filename) + + self.assertEqual(data, file_copy.read()) + self.assertTrue(file_copy.initialize()) + expected_metadata['extra'] = 'fresher' + self.assertDictEqual(expected_metadata, file_copy.metadata) + resp_headers = dict(file_copy.conn.response.getheaders()) + # if k is in expected_headers then we can assert its new value + for k, v in expected_headers.items(): + v = extra_hdrs.get(k, v) + self.assertIn(k.lower(), resp_headers) + self.assertEqual(v, resp_headers[k.lower()]) + + # repeat copy with X-Fresh-Metadata header - existing user + # metadata should not be copied, new completely replaces it. + extra_hdrs = {'Content-Type': 'application/test-updated', + 'X-Object-Meta-Extra': 'fresher', + 'X-Fresh-Metadata': 'true'} + self.assertTrue(file_item.copy( + '%s%s' % (prefix, cont), dest_filename, hdrs=extra_hdrs)) + + self.assertIn(dest_filename, cont.files()) + + file_copy = cont.file(dest_filename) + + self.assertEqual(data, file_copy.read()) + self.assertTrue(file_copy.initialize()) + self.assertEqual('application/test-updated', + file_copy.content_type) + expected_metadata = {'extra': 'fresher'} + self.assertDictEqual(expected_metadata, file_copy.metadata) + resp_headers = dict(file_copy.conn.response.getheaders()) + for k in ('Content-Disposition', 'Content-Encoding'): + self.assertNotIn(k.lower(), resp_headers) def testCopyAccount(self): # makes sure to test encoded characters diff --git a/test/unit/common/middleware/test_copy.py b/test/unit/common/middleware/test_copy.py index 3a6663db00..4c2643dd92 100644 --- a/test/unit/common/middleware/test_copy.py +++ b/test/unit/common/middleware/test_copy.py @@ -1198,6 +1198,218 @@ class TestServerSideCopyMiddleware(unittest.TestCase): self.assertEqual('OPTIONS', self.authorized[0].method) self.assertEqual('/v1/a/c/o', self.authorized[0].path) + def _test_COPY_source_headers(self, extra_put_headers): + # helper method to perform a COPY with some metadata headers that + # should always be sent to the destination + put_headers = {'Destination': '/c1/o', + 'X-Object-Meta-Test2': 'added', + 'X-Object-Sysmeta-Test2': 'added', + 'X-Object-Transient-Sysmeta-Test2': 'added'} + put_headers.update(extra_put_headers) + get_resp_headers = { + 'X-Timestamp': '1234567890.12345', + 'X-Backend-Timestamp': '1234567890.12345', + 'Content-Type': 'text/original', + 'Content-Encoding': 'gzip', + 'Content-Disposition': 'attachment; filename=myfile', + 'X-Object-Meta-Test': 'original', + 'X-Object-Sysmeta-Test': 'original', + 'X-Object-Transient-Sysmeta-Test': 'original', + 'X-Foo': 'Bar'} + self.app.register( + 'GET', '/v1/a/c/o', swob.HTTPOk, headers=get_resp_headers) + self.app.register('PUT', '/v1/a/c1/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o', method='COPY', headers=put_headers) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '201 Created') + calls = self.app.calls_with_headers + self.assertEqual(2, len(calls)) + method, path, req_headers = calls[1] + self.assertEqual('PUT', method) + # these headers should always be applied to the destination + self.assertEqual('added', req_headers.get('X-Object-Meta-Test2')) + self.assertEqual('added', req_headers.get('X-Object-Sysmeta-Test2')) + self.assertEqual('added', + req_headers.get('X-Object-Transient-Sysmeta-Test2')) + return req_headers + + def test_COPY_source_headers_no_updates(self): + # copy should preserve existing metadata if not updated + req_headers = self._test_COPY_source_headers({}) + self.assertEqual('text/original', req_headers.get('Content-Type')) + self.assertEqual('gzip', req_headers.get('Content-Encoding')) + self.assertEqual('attachment; filename=myfile', + req_headers.get('Content-Disposition')) + self.assertEqual('original', req_headers.get('X-Object-Meta-Test')) + self.assertEqual('original', req_headers.get('X-Object-Sysmeta-Test')) + self.assertEqual('original', + req_headers.get('X-Object-Transient-Sysmeta-Test')) + self.assertEqual('Bar', req_headers.get('X-Foo')) + self.assertNotIn('X-Timestamp', req_headers) + self.assertNotIn('X-Backend-Timestamp', req_headers) + + def test_COPY_source_headers_with_updates(self): + # copy should apply any updated values to existing metadata + put_headers = { + 'Content-Type': 'text/not_original', + 'Content-Encoding': 'not_gzip', + 'Content-Disposition': 'attachment; filename=notmyfile', + 'X-Object-Meta-Test': 'not_original', + 'X-Object-Sysmeta-Test': 'not_original', + 'X-Object-Transient-Sysmeta-Test': 'not_original', + 'X-Foo': 'Not Bar'} + req_headers = self._test_COPY_source_headers(put_headers) + self.assertEqual('text/not_original', req_headers.get('Content-Type')) + self.assertEqual('not_gzip', req_headers.get('Content-Encoding')) + self.assertEqual('attachment; filename=notmyfile', + req_headers.get('Content-Disposition')) + self.assertEqual('not_original', req_headers.get('X-Object-Meta-Test')) + self.assertEqual('not_original', + req_headers.get('X-Object-Sysmeta-Test')) + self.assertEqual('not_original', + req_headers.get('X-Object-Transient-Sysmeta-Test')) + self.assertEqual('Not Bar', req_headers.get('X-Foo')) + self.assertNotIn('X-Timestamp', req_headers) + self.assertNotIn('X-Backend-Timestamp', req_headers) + + def test_COPY_x_fresh_metadata_no_updates(self): + # existing user metadata should not be copied, sysmeta is copied + put_headers = { + 'X-Fresh-Metadata': 'true', + 'X-Extra': 'Fresh'} + req_headers = self._test_COPY_source_headers(put_headers) + self.assertEqual('text/original', req_headers.get('Content-Type')) + self.assertEqual('Fresh', req_headers.get('X-Extra')) + self.assertEqual('original', + req_headers.get('X-Object-Sysmeta-Test')) + self.assertIn('X-Fresh-Metadata', req_headers) + self.assertNotIn('X-Object-Meta-Test', req_headers) + self.assertNotIn('X-Object-Transient-Sysmeta-Test', req_headers) + self.assertNotIn('X-Timestamp', req_headers) + self.assertNotIn('X-Backend-Timestamp', req_headers) + self.assertNotIn('Content-Encoding', req_headers) + self.assertNotIn('Content-Disposition', req_headers) + self.assertNotIn('X-Foo', req_headers) + + def test_COPY_x_fresh_metadata_with_updates(self): + # existing user metadata should not be copied, new metadata replaces it + put_headers = { + 'X-Fresh-Metadata': 'true', + 'Content-Type': 'text/not_original', + 'Content-Encoding': 'not_gzip', + 'Content-Disposition': 'attachment; filename=notmyfile', + 'X-Object-Meta-Test': 'not_original', + 'X-Object-Sysmeta-Test': 'not_original', + 'X-Object-Transient-Sysmeta-Test': 'not_original', + 'X-Foo': 'Not Bar', + 'X-Extra': 'Fresh'} + req_headers = self._test_COPY_source_headers(put_headers) + self.assertEqual('Fresh', req_headers.get('X-Extra')) + self.assertEqual('text/not_original', req_headers.get('Content-Type')) + self.assertEqual('not_gzip', req_headers.get('Content-Encoding')) + self.assertEqual('attachment; filename=notmyfile', + req_headers.get('Content-Disposition')) + self.assertEqual('not_original', req_headers.get('X-Object-Meta-Test')) + self.assertEqual('not_original', + req_headers.get('X-Object-Sysmeta-Test')) + self.assertEqual('not_original', + req_headers.get('X-Object-Transient-Sysmeta-Test')) + self.assertEqual('Not Bar', req_headers.get('X-Foo')) + + def _test_POST_source_headers(self, extra_post_headers): + # helper method to perform a POST with metadata headers that should + # always be sent to the destination + post_headers = {'X-Object-Meta-Test2': 'added', + 'X-Object-Sysmeta-Test2': 'added', + 'X-Object-Transient-Sysmeta-Test2': 'added'} + post_headers.update(extra_post_headers) + get_resp_headers = { + 'X-Timestamp': '1234567890.12345', + 'X-Backend-Timestamp': '1234567890.12345', + 'Content-Type': 'text/original', + 'Content-Encoding': 'gzip', + 'Content-Disposition': 'attachment; filename=myfile', + 'X-Object-Meta-Test': 'original', + 'X-Object-Sysmeta-Test': 'original', + 'X-Object-Transient-Sysmeta-Test': 'original', + 'X-Foo': 'Bar'} + self.app.register( + 'GET', '/v1/a/c/o', swob.HTTPOk, headers=get_resp_headers) + self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/o', method='POST', headers=post_headers) + status, headers, body = self.call_ssc(req) + self.assertEqual(status, '202 Accepted') + calls = self.app.calls_with_headers + self.assertEqual(2, len(calls)) + method, path, req_headers = calls[1] + self.assertEqual('PUT', method) + # these headers should always be applied to the destination + self.assertEqual('added', req_headers.get('X-Object-Meta-Test2')) + self.assertEqual('added', + req_headers.get('X-Object-Transient-Sysmeta-Test2')) + # POSTed sysmeta should never be applied to the destination + self.assertNotIn('X-Object-Sysmeta-Test2', req_headers) + # existing sysmeta should always be preserved + self.assertEqual('original', + req_headers.get('X-Object-Sysmeta-Test')) + return req_headers + + def test_POST_no_updates(self): + post_headers = {} + req_headers = self._test_POST_source_headers(post_headers) + self.assertEqual('text/original', req_headers.get('Content-Type')) + self.assertNotIn('X-Object-Meta-Test', req_headers) + self.assertNotIn('X-Object-Transient-Sysmeta-Test', req_headers) + self.assertNotIn('X-Timestamp', req_headers) + self.assertNotIn('X-Backend-Timestamp', req_headers) + self.assertNotIn('Content-Encoding', req_headers) + self.assertNotIn('Content-Disposition', req_headers) + self.assertNotIn('X-Foo', req_headers) + + def test_POST_with_updates(self): + post_headers = { + 'Content-Type': 'text/not_original', + 'Content-Encoding': 'not_gzip', + 'Content-Disposition': 'attachment; filename=notmyfile', + 'X-Object-Meta-Test': 'not_original', + 'X-Object-Sysmeta-Test': 'not_original', + 'X-Object-Transient-Sysmeta-Test': 'not_original', + 'X-Foo': 'Not Bar', + } + req_headers = self._test_POST_source_headers(post_headers) + self.assertEqual('text/not_original', req_headers.get('Content-Type')) + self.assertEqual('not_gzip', req_headers.get('Content-Encoding')) + self.assertEqual('attachment; filename=notmyfile', + req_headers.get('Content-Disposition')) + self.assertEqual('not_original', req_headers.get('X-Object-Meta-Test')) + self.assertEqual('not_original', + req_headers.get('X-Object-Transient-Sysmeta-Test')) + self.assertEqual('Not Bar', req_headers.get('X-Foo')) + + def test_POST_x_fresh_metadata_with_updates(self): + # post-as-copy trumps x-fresh-metadata i.e. existing user metadata + # should not be copied, sysmeta is copied *and not updated with new* + post_headers = { + 'X-Fresh-Metadata': 'true', + 'Content-Type': 'text/not_original', + 'Content-Encoding': 'not_gzip', + 'Content-Disposition': 'attachment; filename=notmyfile', + 'X-Object-Meta-Test': 'not_original', + 'X-Object-Sysmeta-Test': 'not_original', + 'X-Object-Transient-Sysmeta-Test': 'not_original', + 'X-Foo': 'Not Bar', + } + req_headers = self._test_POST_source_headers(post_headers) + self.assertEqual('text/not_original', req_headers.get('Content-Type')) + self.assertEqual('not_gzip', req_headers.get('Content-Encoding')) + self.assertEqual('attachment; filename=notmyfile', + req_headers.get('Content-Disposition')) + self.assertEqual('not_original', req_headers.get('X-Object-Meta-Test')) + self.assertEqual('not_original', + req_headers.get('X-Object-Transient-Sysmeta-Test')) + self.assertEqual('Not Bar', req_headers.get('X-Foo')) + self.assertIn('X-Fresh-Metadata', req_headers) + class TestServerSideCopyConfiguration(unittest.TestCase):