openstack/swift
Code Issues Proposed changes

container-sync: Sync static links similar to how we sync SLOs

Browse Source 
This allows static symlinks to be synced before their target. Dynamic
symlinks could already be synced even if target object has not been
synced, but static links previously required that target object existed
before it can be PUT. Now, have container_sync middleware plumb in an
override like it does for SLO.

Change-Id: I3bfc62b77b247003adcee6bd4d374168bfd6707d
This commit is contained in:
Thiago da Silva 2019-12-10 16:32:47 +11:00 committed by Tim Burke
parent 9548111e24
commit 26ff2eb1cb
9 changed files with 238 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
doc/source/overview_container_sync.rst
View File

@ -45,6 +45,12 @@ synchronization key.
are being synced, then you should follow the instructions for
:ref:`symlink_container_sync_client_config` to be compatible with symlinks.
Be aware that symlinks may be synced before their targets even if they are
in the same container and were created after the target objects. In such
cases, a GET for the symlink will fail with a ``404 Not Found`` error. If
the target has been overwritten, a GET may produce an older version (for
dynamic links) or a ``409 Conflict`` error (for static links).
--------------------------
Configuring Container Sync
--------------------------

29
swift/common/middleware/container_sync.py
View File

@ -15,6 +15,7 @@
import os
from swift.common.constraints import valid_api_version
from swift.common.container_sync_realms import ContainerSyncRealms
from swift.common.swob import HTTPBadRequest, HTTPUnauthorized, wsgify
from swift.common.utils import (
@ -67,8 +68,27 @@ class ContainerSync(object):
@wsgify
def __call__(self, req):
if req.path == '/info':
# Ensure /info requests get the freshest results
self.register_info()
return self.app
try:
(version, acc, cont, obj) = req.split_path(3, 4, True)
bad_path = False
except ValueError:
bad_path = True
# use of bad_path bool is to avoid recursive tracebacks
if bad_path or not valid_api_version(version):
return self.app
# validate container-sync metdata update
info = get_container_info(
req.environ, self.app, swift_source='CS')
sync_to = req.headers.get('x-container-sync-to')
if not self.allow_full_urls:
sync_to = req.headers.get('x-container-sync-to')
if sync_to and not sync_to.startswith('//'):
raise HTTPBadRequest(
body='Full URLs are not allowed for X-Container-Sync-To '
@ -90,8 +110,6 @@ class ContainerSync(object):
req.environ.setdefault('swift.log_info', []).append(
'cs:no-local-realm-key')
else:
info = get_container_info(
req.environ, self.app, swift_source='CS')
user_key = info.get('sync_key')
if not user_key:
req.environ.setdefault('swift.log_info', []).append(
@ -134,10 +152,9 @@ class ContainerSync(object):
# syntax and might be synced before its segments, so stop SLO
# middleware from performing the usual manifest validation.
req.environ['swift.slo_override'] = True
# Similar arguments for static symlinks
req.environ['swift.symlink_override'] = True
if req.path == '/info':
# Ensure /info requests get the freshest results
self.register_info()
return self.app

6
swift/common/middleware/symlink.py
View File

@ -506,6 +506,12 @@ class SymlinkObjectContext(WSGIContext):
def _validate_etag_and_update_sysmeta(self, req, symlink_target_path,
etag):
if req.environ.get('swift.symlink_override'):
req.headers[TGT_ETAG_SYSMETA_SYMLINK_HDR] = etag
req.headers[TGT_BYTES_SYSMETA_SYMLINK_HDR] = \
req.headers[TGT_BYTES_SYMLINK_HDR]
return
# next we'll make sure the E-Tag matches a real object
new_req = make_subrequest(
req.environ, path=wsgi_quote(symlink_target_path), method='HEAD',

1
swift/proxy/controllers/base.py
View File

@ -179,6 +179,7 @@ def headers_to_container_info(headers, status_int=HTTP_OK):
'status': status_int,
'read_acl': headers.get('x-container-read'),
'write_acl': headers.get('x-container-write'),
'sync_to': headers.get('x-container-sync-to'),
'sync_key': headers.get('x-container-sync-key'),
'object_count': headers.get('x-container-object-count'),
'bytes': headers.get('x-container-bytes-used'),

3
test/probe/test_container_failures.py
View File

@ -147,6 +147,9 @@ class TestContainerFailures(ReplProbeTest):
def run_test(num_locks, catch_503):
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
# Get the container info into memcache (so no stray
# get_container_info calls muck up our timings)
client.get_container(self.url, self.token, container)
db_files = self._get_container_db_files(container)
db_conns = []
for i in range(num_locks):

176
test/probe/test_container_sync.py
View File

@ -562,6 +562,182 @@ class TestContainerSyncAndSymlink(BaseTestContainerSync):
self.url, self.token, dest_container, symlink_name)
self.assertEqual(target_body, actual_target_body)
def test_sync_static_symlink_different_container(self):
source_container, dest_container = self._setup_synced_containers()
symlink_cont = 'symlink-container-%s' % uuid.uuid4()
client.put_container(self.url, self.token, symlink_cont)
# upload a target to symlink container
target_name = 'target-%s' % uuid.uuid4()
target_body = b'target body'
etag = client.put_object(
self.url, self.token, symlink_cont, target_name,
target_body)
# upload a regular object
regular_name = 'regular-%s' % uuid.uuid4()
regular_body = b'regular body'
client.put_object(
self.url, self.token, source_container, regular_name,
regular_body)
# static symlink
target_path = '%s/%s' % (symlink_cont, target_name)
symlink_name = 'symlink-%s' % uuid.uuid4()
put_headers = {'X-Symlink-Target': target_path,
'X-Symlink-Target-Etag': etag}
# upload the symlink
client.put_object(
self.url, self.token, source_container, symlink_name,
'', headers=put_headers)
# verify object is a symlink
resp_headers, symlink_body = client.get_object(
self.url, self.token, source_container, symlink_name,
query_string='symlink=get')
self.assertEqual(b'', symlink_body)
self.assertIn('x-symlink-target', resp_headers)
self.assertIn('x-symlink-target-etag', resp_headers)
# verify symlink behavior
resp_headers, actual_target_body = client.get_object(
self.url, self.token, source_container, symlink_name)
self.assertEqual(target_body, actual_target_body)
self.assertIn('content-location', resp_headers)
content_location = resp_headers['content-location']
# cycle container-sync
Manager(['container-sync']).once()
# regular object should have synced
resp_headers, actual_target_body = client.get_object(
self.url, self.token, dest_container, regular_name)
self.assertEqual(regular_body, actual_target_body)
# static symlink gets synced, too
resp_headers, actual_target_body = client.get_object(
self.url, self.token, dest_container, symlink_name)
self.assertEqual(target_body, actual_target_body)
self.assertIn('content-location', resp_headers)
self.assertEqual(content_location, resp_headers['content-location'])
def test_sync_busted_static_symlink_different_container(self):
source_container, dest_container = self._setup_synced_containers()
symlink_cont = 'symlink-container-%s' % uuid.uuid4()
client.put_container(self.url, self.token, symlink_cont)
# upload a target to symlink container
target_name = 'target-%s' % uuid.uuid4()
target_body = b'target body'
etag = client.put_object(
self.url, self.token, symlink_cont, target_name,
target_body)
# upload a regular object
regular_name = 'regular-%s' % uuid.uuid4()
regular_body = b'regular body'
client.put_object(
self.url, self.token, source_container, regular_name,
regular_body)
# static symlink
target_path = '%s/%s' % (symlink_cont, target_name)
symlink_name = 'symlink-%s' % uuid.uuid4()
put_headers = {'X-Symlink-Target': target_path,
'X-Symlink-Target-Etag': etag}
# upload the symlink
client.put_object(
self.url, self.token, source_container, symlink_name,
'', headers=put_headers)
# verify object is a symlink
resp_headers, symlink_body = client.get_object(
self.url, self.token, source_container, symlink_name,
query_string='symlink=get')
self.assertEqual(b'', symlink_body)
self.assertIn('x-symlink-target', resp_headers)
self.assertIn('x-symlink-target-etag', resp_headers)
# verify symlink behavior
resp_headers, actual_target_body = client.get_object(
self.url, self.token, source_container, symlink_name)
self.assertEqual(target_body, actual_target_body)
self.assertIn('content-location', resp_headers)
content_location = resp_headers['content-location']
# Break the link
client.put_object(
self.url, self.token, symlink_cont, target_name,
b'something else')
# cycle container-sync
Manager(['container-sync']).once()
# regular object should have synced
resp_headers, actual_target_body = client.get_object(
self.url, self.token, dest_container, regular_name)
self.assertEqual(regular_body, actual_target_body)
# static symlink gets synced, too, even though the target's different!
with self.assertRaises(ClientException) as cm:
client.get_object(
self.url, self.token, dest_container, symlink_name)
self.assertEqual(409, cm.exception.http_status)
resp_headers = cm.exception.http_response_headers
self.assertIn('content-location', resp_headers)
self.assertEqual(content_location, resp_headers['content-location'])
def test_sync_static_symlink(self):
source_container, dest_container = self._setup_synced_containers()
# upload a target to symlink container
target_name = 'target-%s' % uuid.uuid4()
target_body = b'target body'
etag = client.put_object(
self.url, self.token, source_container, target_name,
target_body)
# static symlink
target_path = '%s/%s' % (source_container, target_name)
symlink_name = 'symlink-%s' % uuid.uuid4()
put_headers = {'X-Symlink-Target': target_path,
'X-Symlink-Target-Etag': etag}
# upload the symlink
client.put_object(
self.url, self.token, source_container, symlink_name,
'', headers=put_headers)
# verify object is a symlink
resp_headers, symlink_body = client.get_object(
self.url, self.token, source_container, symlink_name,
query_string='symlink=get')
self.assertEqual(b'', symlink_body)
self.assertIn('x-symlink-target', resp_headers)
self.assertIn('x-symlink-target-etag', resp_headers)
# verify symlink behavior
resp_headers, actual_target_body = client.get_object(
self.url, self.token, source_container, symlink_name)
self.assertEqual(target_body, actual_target_body)
# cycle container-sync
Manager(['container-sync']).once()
# regular object should have synced
resp_headers, actual_target_body = client.get_object(
self.url, self.token, dest_container, target_name)
self.assertEqual(target_body, actual_target_body)
# and static link too
resp_headers, actual_target_body = client.get_object(
self.url, self.token, dest_container, symlink_name)
self.assertEqual(target_body, actual_target_body)
if __name__ == "__main__":
unittest.main()

3
test/unit/common/middleware/test_container_sync.py
View File

@ -216,6 +216,7 @@ cluster_dfw1 = http://dfw1.host/v1/
self.assertIn('cs:invalid-sig', req.environ.get('swift.log_info'))
self.assertNotIn('swift.authorize_override', req.environ)
self.assertNotIn('swift.slo_override', req.environ)
self.assertNotIn('swift.symlink_override', req.environ)
def test_valid_sig(self):
ts = '1455221706.726999_0123456789abcdef'
@ -235,6 +236,7 @@ cluster_dfw1 = http://dfw1.host/v1/
self.assertEqual(ts, resp.headers['X-Timestamp'])
self.assertIn('swift.authorize_override', req.environ)
self.assertIn('swift.slo_override', req.environ)
self.assertIn('swift.symlink_override', req.environ)
def test_valid_sig2(self):
sig = self.sync.realms_conf.get_sig(
@ -250,6 +252,7 @@ cluster_dfw1 = http://dfw1.host/v1/
self.assertIn('cs:valid', req.environ.get('swift.log_info'))
self.assertIn('swift.authorize_override', req.environ)
self.assertIn('swift.slo_override', req.environ)
self.assertIn('swift.symlink_override', req.environ)
def test_info(self):
req = swob.Request.blank('/info')

15
test/unit/common/middleware/test_symlink.py
View File

@ -243,6 +243,21 @@ class TestSymlinkMiddleware(TestSymlinkMiddlewareBase):
# ... we better have a body!
self.assertIn(b'Internal Error', body)
def test_symlink_simple_put_to_non_existing_object_override(self):
self.app.register('HEAD', '/v1/a/c1/o', swob.HTTPNotFound, {})
self.app.register('PUT', '/v1/a/c/symlink', swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/symlink', method='PUT',
headers={
'X-Symlink-Target': 'c1/o',
'X-Symlink-Target-Etag': 'some-tgt-etag',
# this header isn't normally sent with PUT
'X-Symlink-Target-Bytes': '13',
}, body='')
# this can be set in container_sync
req.environ['swift.symlink_override'] = True
status, headers, body = self.call_sym(req)
self.assertEqual(status, '201 Created')
def test_symlink_put_with_prevalidated_etag(self):
self.app.register('PUT', '/v1/a/c/symlink', swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/symlink', method='PUT', headers={

5
test/unit/proxy/controllers/test_base.py
View File

@ -639,6 +639,8 @@ class TestFuncs(unittest.TestCase):
self.assertEqual(resp['status'], 404)
self.assertIsNone(resp['read_acl'])
self.assertIsNone(resp['write_acl'])
self.assertIsNone(resp['sync_key'])
self.assertIsNone(resp['sync_to'])
def test_headers_to_container_info_meta(self):
headers = {'X-Container-Meta-Whatevs': 14,
@ -662,11 +664,14 @@ class TestFuncs(unittest.TestCase):
'x-container-read': 'readvalue',
'x-container-write': 'writevalue',
'x-container-sync-key': 'keyvalue',
'x-container-sync-to': '//r/c/a/c',
'x-container-meta-access-control-allow-origin': 'here',
}
resp = headers_to_container_info(headers.items(), 200)
self.assertEqual(resp['read_acl'], 'readvalue')
self.assertEqual(resp['write_acl'], 'writevalue')
self.assertEqual(resp['sync_key'], 'keyvalue')
self.assertEqual(resp['sync_to'], '//r/c/a/c')
self.assertEqual(resp['cors']['allow_origin'], 'here')
headers['x-unused-header'] = 'blahblahblah'