From 26ff2eb1cbec8052797dc5e28af3a8049d5e5dfb Mon Sep 17 00:00:00 2001 From: Thiago da Silva Date: Tue, 10 Dec 2019 16:32:47 +1100 Subject: [PATCH] container-sync: Sync static links similar to how we sync SLOs This allows static symlinks to be synced before their target. Dynamic symlinks could already be synced even if target object has not been synced, but static links previously required that target object existed before it can be PUT. Now, have container_sync middleware plumb in an override like it does for SLO. Change-Id: I3bfc62b77b247003adcee6bd4d374168bfd6707d --- doc/source/overview_container_sync.rst | 6 + swift/common/middleware/container_sync.py | 29 ++- swift/common/middleware/symlink.py | 6 + swift/proxy/controllers/base.py | 1 + test/probe/test_container_failures.py | 3 + test/probe/test_container_sync.py | 176 ++++++++++++++++++ .../common/middleware/test_container_sync.py | 3 + test/unit/common/middleware/test_symlink.py | 15 ++ test/unit/proxy/controllers/test_base.py | 5 + 9 files changed, 238 insertions(+), 6 deletions(-) diff --git a/doc/source/overview_container_sync.rst b/doc/source/overview_container_sync.rst index ceabfd527e..7413911e87 100644 --- a/doc/source/overview_container_sync.rst +++ b/doc/source/overview_container_sync.rst @@ -45,6 +45,12 @@ synchronization key. are being synced, then you should follow the instructions for :ref:`symlink_container_sync_client_config` to be compatible with symlinks. + Be aware that symlinks may be synced before their targets even if they are + in the same container and were created after the target objects. In such + cases, a GET for the symlink will fail with a ``404 Not Found`` error. If + the target has been overwritten, a GET may produce an older version (for + dynamic links) or a ``409 Conflict`` error (for static links). + -------------------------- Configuring Container Sync -------------------------- diff --git a/swift/common/middleware/container_sync.py b/swift/common/middleware/container_sync.py index f005a40a43..bde33ca70c 100644 --- a/swift/common/middleware/container_sync.py +++ b/swift/common/middleware/container_sync.py @@ -15,6 +15,7 @@ import os +from swift.common.constraints import valid_api_version from swift.common.container_sync_realms import ContainerSyncRealms from swift.common.swob import HTTPBadRequest, HTTPUnauthorized, wsgify from swift.common.utils import ( @@ -67,8 +68,27 @@ class ContainerSync(object): @wsgify def __call__(self, req): + if req.path == '/info': + # Ensure /info requests get the freshest results + self.register_info() + return self.app + + try: + (version, acc, cont, obj) = req.split_path(3, 4, True) + bad_path = False + except ValueError: + bad_path = True + + # use of bad_path bool is to avoid recursive tracebacks + if bad_path or not valid_api_version(version): + return self.app + + # validate container-sync metdata update + info = get_container_info( + req.environ, self.app, swift_source='CS') + sync_to = req.headers.get('x-container-sync-to') + if not self.allow_full_urls: - sync_to = req.headers.get('x-container-sync-to') if sync_to and not sync_to.startswith('//'): raise HTTPBadRequest( body='Full URLs are not allowed for X-Container-Sync-To ' @@ -90,8 +110,6 @@ class ContainerSync(object): req.environ.setdefault('swift.log_info', []).append( 'cs:no-local-realm-key') else: - info = get_container_info( - req.environ, self.app, swift_source='CS') user_key = info.get('sync_key') if not user_key: req.environ.setdefault('swift.log_info', []).append( @@ -134,10 +152,9 @@ class ContainerSync(object): # syntax and might be synced before its segments, so stop SLO # middleware from performing the usual manifest validation. req.environ['swift.slo_override'] = True + # Similar arguments for static symlinks + req.environ['swift.symlink_override'] = True - if req.path == '/info': - # Ensure /info requests get the freshest results - self.register_info() return self.app diff --git a/swift/common/middleware/symlink.py b/swift/common/middleware/symlink.py index 0dc75419e9..1854e712ad 100644 --- a/swift/common/middleware/symlink.py +++ b/swift/common/middleware/symlink.py @@ -506,6 +506,12 @@ class SymlinkObjectContext(WSGIContext): def _validate_etag_and_update_sysmeta(self, req, symlink_target_path, etag): + if req.environ.get('swift.symlink_override'): + req.headers[TGT_ETAG_SYSMETA_SYMLINK_HDR] = etag + req.headers[TGT_BYTES_SYSMETA_SYMLINK_HDR] = \ + req.headers[TGT_BYTES_SYMLINK_HDR] + return + # next we'll make sure the E-Tag matches a real object new_req = make_subrequest( req.environ, path=wsgi_quote(symlink_target_path), method='HEAD', diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index 87da0aa5fc..831864aba1 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -179,6 +179,7 @@ def headers_to_container_info(headers, status_int=HTTP_OK): 'status': status_int, 'read_acl': headers.get('x-container-read'), 'write_acl': headers.get('x-container-write'), + 'sync_to': headers.get('x-container-sync-to'), 'sync_key': headers.get('x-container-sync-key'), 'object_count': headers.get('x-container-object-count'), 'bytes': headers.get('x-container-bytes-used'), diff --git a/test/probe/test_container_failures.py b/test/probe/test_container_failures.py index 6d8156088e..3f87f9b57f 100644 --- a/test/probe/test_container_failures.py +++ b/test/probe/test_container_failures.py @@ -147,6 +147,9 @@ class TestContainerFailures(ReplProbeTest): def run_test(num_locks, catch_503): container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) + # Get the container info into memcache (so no stray + # get_container_info calls muck up our timings) + client.get_container(self.url, self.token, container) db_files = self._get_container_db_files(container) db_conns = [] for i in range(num_locks): diff --git a/test/probe/test_container_sync.py b/test/probe/test_container_sync.py index f04941c444..96f63065c6 100644 --- a/test/probe/test_container_sync.py +++ b/test/probe/test_container_sync.py @@ -562,6 +562,182 @@ class TestContainerSyncAndSymlink(BaseTestContainerSync): self.url, self.token, dest_container, symlink_name) self.assertEqual(target_body, actual_target_body) + def test_sync_static_symlink_different_container(self): + source_container, dest_container = self._setup_synced_containers() + + symlink_cont = 'symlink-container-%s' % uuid.uuid4() + client.put_container(self.url, self.token, symlink_cont) + + # upload a target to symlink container + target_name = 'target-%s' % uuid.uuid4() + target_body = b'target body' + etag = client.put_object( + self.url, self.token, symlink_cont, target_name, + target_body) + + # upload a regular object + regular_name = 'regular-%s' % uuid.uuid4() + regular_body = b'regular body' + client.put_object( + self.url, self.token, source_container, regular_name, + regular_body) + + # static symlink + target_path = '%s/%s' % (symlink_cont, target_name) + symlink_name = 'symlink-%s' % uuid.uuid4() + put_headers = {'X-Symlink-Target': target_path, + 'X-Symlink-Target-Etag': etag} + + # upload the symlink + client.put_object( + self.url, self.token, source_container, symlink_name, + '', headers=put_headers) + + # verify object is a symlink + resp_headers, symlink_body = client.get_object( + self.url, self.token, source_container, symlink_name, + query_string='symlink=get') + self.assertEqual(b'', symlink_body) + self.assertIn('x-symlink-target', resp_headers) + self.assertIn('x-symlink-target-etag', resp_headers) + + # verify symlink behavior + resp_headers, actual_target_body = client.get_object( + self.url, self.token, source_container, symlink_name) + self.assertEqual(target_body, actual_target_body) + self.assertIn('content-location', resp_headers) + content_location = resp_headers['content-location'] + + # cycle container-sync + Manager(['container-sync']).once() + + # regular object should have synced + resp_headers, actual_target_body = client.get_object( + self.url, self.token, dest_container, regular_name) + self.assertEqual(regular_body, actual_target_body) + + # static symlink gets synced, too + resp_headers, actual_target_body = client.get_object( + self.url, self.token, dest_container, symlink_name) + self.assertEqual(target_body, actual_target_body) + self.assertIn('content-location', resp_headers) + self.assertEqual(content_location, resp_headers['content-location']) + + def test_sync_busted_static_symlink_different_container(self): + source_container, dest_container = self._setup_synced_containers() + + symlink_cont = 'symlink-container-%s' % uuid.uuid4() + client.put_container(self.url, self.token, symlink_cont) + + # upload a target to symlink container + target_name = 'target-%s' % uuid.uuid4() + target_body = b'target body' + etag = client.put_object( + self.url, self.token, symlink_cont, target_name, + target_body) + + # upload a regular object + regular_name = 'regular-%s' % uuid.uuid4() + regular_body = b'regular body' + client.put_object( + self.url, self.token, source_container, regular_name, + regular_body) + + # static symlink + target_path = '%s/%s' % (symlink_cont, target_name) + symlink_name = 'symlink-%s' % uuid.uuid4() + put_headers = {'X-Symlink-Target': target_path, + 'X-Symlink-Target-Etag': etag} + + # upload the symlink + client.put_object( + self.url, self.token, source_container, symlink_name, + '', headers=put_headers) + + # verify object is a symlink + resp_headers, symlink_body = client.get_object( + self.url, self.token, source_container, symlink_name, + query_string='symlink=get') + self.assertEqual(b'', symlink_body) + self.assertIn('x-symlink-target', resp_headers) + self.assertIn('x-symlink-target-etag', resp_headers) + + # verify symlink behavior + resp_headers, actual_target_body = client.get_object( + self.url, self.token, source_container, symlink_name) + self.assertEqual(target_body, actual_target_body) + self.assertIn('content-location', resp_headers) + content_location = resp_headers['content-location'] + + # Break the link + client.put_object( + self.url, self.token, symlink_cont, target_name, + b'something else') + + # cycle container-sync + Manager(['container-sync']).once() + + # regular object should have synced + resp_headers, actual_target_body = client.get_object( + self.url, self.token, dest_container, regular_name) + self.assertEqual(regular_body, actual_target_body) + + # static symlink gets synced, too, even though the target's different! + with self.assertRaises(ClientException) as cm: + client.get_object( + self.url, self.token, dest_container, symlink_name) + self.assertEqual(409, cm.exception.http_status) + resp_headers = cm.exception.http_response_headers + self.assertIn('content-location', resp_headers) + self.assertEqual(content_location, resp_headers['content-location']) + + def test_sync_static_symlink(self): + source_container, dest_container = self._setup_synced_containers() + + # upload a target to symlink container + target_name = 'target-%s' % uuid.uuid4() + target_body = b'target body' + etag = client.put_object( + self.url, self.token, source_container, target_name, + target_body) + + # static symlink + target_path = '%s/%s' % (source_container, target_name) + symlink_name = 'symlink-%s' % uuid.uuid4() + put_headers = {'X-Symlink-Target': target_path, + 'X-Symlink-Target-Etag': etag} + + # upload the symlink + client.put_object( + self.url, self.token, source_container, symlink_name, + '', headers=put_headers) + + # verify object is a symlink + resp_headers, symlink_body = client.get_object( + self.url, self.token, source_container, symlink_name, + query_string='symlink=get') + self.assertEqual(b'', symlink_body) + self.assertIn('x-symlink-target', resp_headers) + self.assertIn('x-symlink-target-etag', resp_headers) + + # verify symlink behavior + resp_headers, actual_target_body = client.get_object( + self.url, self.token, source_container, symlink_name) + self.assertEqual(target_body, actual_target_body) + + # cycle container-sync + Manager(['container-sync']).once() + + # regular object should have synced + resp_headers, actual_target_body = client.get_object( + self.url, self.token, dest_container, target_name) + self.assertEqual(target_body, actual_target_body) + + # and static link too + resp_headers, actual_target_body = client.get_object( + self.url, self.token, dest_container, symlink_name) + self.assertEqual(target_body, actual_target_body) + if __name__ == "__main__": unittest.main() diff --git a/test/unit/common/middleware/test_container_sync.py b/test/unit/common/middleware/test_container_sync.py index ec030d7490..15e33dce55 100644 --- a/test/unit/common/middleware/test_container_sync.py +++ b/test/unit/common/middleware/test_container_sync.py @@ -216,6 +216,7 @@ cluster_dfw1 = http://dfw1.host/v1/ self.assertIn('cs:invalid-sig', req.environ.get('swift.log_info')) self.assertNotIn('swift.authorize_override', req.environ) self.assertNotIn('swift.slo_override', req.environ) + self.assertNotIn('swift.symlink_override', req.environ) def test_valid_sig(self): ts = '1455221706.726999_0123456789abcdef' @@ -235,6 +236,7 @@ cluster_dfw1 = http://dfw1.host/v1/ self.assertEqual(ts, resp.headers['X-Timestamp']) self.assertIn('swift.authorize_override', req.environ) self.assertIn('swift.slo_override', req.environ) + self.assertIn('swift.symlink_override', req.environ) def test_valid_sig2(self): sig = self.sync.realms_conf.get_sig( @@ -250,6 +252,7 @@ cluster_dfw1 = http://dfw1.host/v1/ self.assertIn('cs:valid', req.environ.get('swift.log_info')) self.assertIn('swift.authorize_override', req.environ) self.assertIn('swift.slo_override', req.environ) + self.assertIn('swift.symlink_override', req.environ) def test_info(self): req = swob.Request.blank('/info') diff --git a/test/unit/common/middleware/test_symlink.py b/test/unit/common/middleware/test_symlink.py index 44ba9b2cf3..c008e23c98 100644 --- a/test/unit/common/middleware/test_symlink.py +++ b/test/unit/common/middleware/test_symlink.py @@ -243,6 +243,21 @@ class TestSymlinkMiddleware(TestSymlinkMiddlewareBase): # ... we better have a body! self.assertIn(b'Internal Error', body) + def test_symlink_simple_put_to_non_existing_object_override(self): + self.app.register('HEAD', '/v1/a/c1/o', swob.HTTPNotFound, {}) + self.app.register('PUT', '/v1/a/c/symlink', swob.HTTPCreated, {}) + req = Request.blank('/v1/a/c/symlink', method='PUT', + headers={ + 'X-Symlink-Target': 'c1/o', + 'X-Symlink-Target-Etag': 'some-tgt-etag', + # this header isn't normally sent with PUT + 'X-Symlink-Target-Bytes': '13', + }, body='') + # this can be set in container_sync + req.environ['swift.symlink_override'] = True + status, headers, body = self.call_sym(req) + self.assertEqual(status, '201 Created') + def test_symlink_put_with_prevalidated_etag(self): self.app.register('PUT', '/v1/a/c/symlink', swob.HTTPCreated, {}) req = Request.blank('/v1/a/c/symlink', method='PUT', headers={ diff --git a/test/unit/proxy/controllers/test_base.py b/test/unit/proxy/controllers/test_base.py index 2f97380bc9..e2aa93487f 100644 --- a/test/unit/proxy/controllers/test_base.py +++ b/test/unit/proxy/controllers/test_base.py @@ -639,6 +639,8 @@ class TestFuncs(unittest.TestCase): self.assertEqual(resp['status'], 404) self.assertIsNone(resp['read_acl']) self.assertIsNone(resp['write_acl']) + self.assertIsNone(resp['sync_key']) + self.assertIsNone(resp['sync_to']) def test_headers_to_container_info_meta(self): headers = {'X-Container-Meta-Whatevs': 14, @@ -662,11 +664,14 @@ class TestFuncs(unittest.TestCase): 'x-container-read': 'readvalue', 'x-container-write': 'writevalue', 'x-container-sync-key': 'keyvalue', + 'x-container-sync-to': '//r/c/a/c', 'x-container-meta-access-control-allow-origin': 'here', } resp = headers_to_container_info(headers.items(), 200) self.assertEqual(resp['read_acl'], 'readvalue') self.assertEqual(resp['write_acl'], 'writevalue') + self.assertEqual(resp['sync_key'], 'keyvalue') + self.assertEqual(resp['sync_to'], '//r/c/a/c') self.assertEqual(resp['cors']['allow_origin'], 'here') headers['x-unused-header'] = 'blahblahblah'