openstack/swift
Code Issues Proposed changes

Allow bulk to fwd some headers at tar extraction

Browse Source 
Whitelisted headers include X-Delete-At/X-Delete-After and all Object
metadata headers (X-Object-Meta-*)

Closes-Bug: 1857546
Change-Id: If5fb164693e395f89d57899fb8ab355f1e3f817c
This commit is contained in:
Romain LE DISEZ 2020-01-09 09:08:25 -05:00 committed by Tim Burke
parent a4f1078864
commit ff0753fe19
2 changed files with 46 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
swift/common/middleware/bulk.py
View File

@ -98,6 +98,10 @@ The bulk middleware will handle xattrs stored by both GNU and BSD tar (2).
Only xattrs ``user.mime_type`` and ``user.meta.*`` are processed. Other
attributes are ignored.
In addition to the extended attributes, the object metadata and the
x-delete-at/x-delete-after headers set in the request are also assigned to the
extracted objects.
Notes:
(1) The POSIX 1003.1-2001 (pax) format. The default format on GNU tar
@ -206,6 +210,7 @@ from swift.common.utils import get_logger, register_swift_info, \
StreamingPile
from swift.common import constraints
from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND, HTTP_CONFLICT
from swift.common.request_helpers import is_user_meta
from swift.common.wsgi import make_subrequest
@ -622,6 +627,12 @@ class Bulk(object):
'X-Auth-Token': req.headers.get('X-Auth-Token'),
}
# Copy some whitelisted headers to the subrequest
for k, v in req.headers.items():
if ((k.lower() in ('x-delete-at', 'x-delete-after'))
or is_user_meta('object', k)):
create_headers[k] = v
create_obj_req = make_subrequest(
req.environ, method='PUT',
path=wsgi_quote(destination),

39
test/unit/common/middleware/test_bulk.py
View File

@ -41,12 +41,21 @@ class FakeApp(object):
def __init__(self):
self.calls = 0
self.delete_paths = []
self.put_paths = []
self.max_pathlen = 100
self.del_cont_total_calls = 2
self.del_cont_cur_call = 0
def __call__(self, env, start_response):
self.calls += 1
if env.get('swift.source') in ('EA', 'BD'):
assert not env.get('swift.proxy_access_log_made')
if not six.PY2:
# Check that it's valid WSGI
assert all(0 <= ord(c) <= 255 for c in env['PATH_INFO'])
if env['REQUEST_METHOD'] == 'PUT':
self.put_paths.append(env['PATH_INFO'])
if env['PATH_INFO'].startswith('/unauth/'):
if env['PATH_INFO'].endswith('/c/f_ok'):
return Response(status='204 No Content')(env, start_response)
@ -224,8 +233,16 @@ class TestUntarMetadata(unittest.TestCase):
req = Request.blank('/v1/a/c?extract-archive=tar')
req.environ['REQUEST_METHOD'] = 'PUT'
req.environ['wsgi.input'] = tar_ball
req.headers['transfer-encoding'] = 'chunked'
req.headers['accept'] = 'application/json;q=1.0'
# Since there should be a proxy-logging left of us...
req.environ['swift.proxy_access_log_made'] = True
req.headers.update({
'transfer-encoding': 'chunked',
'accept': 'application/json;q=1.0',
'X-Delete-At': '1577383915',
'X-Object-Meta-Dog': 'Rantanplan',
'X-Horse': 'Jolly Jumper',
'X-Object-Meta-Cat': 'tabby',
})
resp = req.get_response(self.bulk)
self.assertEqual(resp.status_int, 200)
@ -244,12 +261,19 @@ class TestUntarMetadata(unittest.TestCase):
self.assertEqual(
put1_headers.get('X-Object-Meta-Afternoon-Snack'),
'gigantic bucket of coffee')
self.assertEqual(put1_headers.get('X-Delete-At'), '1577383915')
self.assertEqual(put1_headers.get('X-Object-Meta-Dog'), 'Rantanplan')
self.assertEqual(put1_headers.get('X-Object-Meta-Cat'), 'tabby')
self.assertIsNone(put1_headers.get('X-Horse'))
put2_headers = HeaderKeyDict(self.app.calls_with_headers[2][2])
self.assertEqual(put2_headers.get('X-Object-Meta-Muppet'), 'bert')
self.assertEqual(put2_headers.get('X-Object-Meta-Cat'), 'fluffy')
self.assertIsNone(put2_headers.get('Content-Type'))
self.assertIsNone(put2_headers.get('X-Object-Meta-Blah'))
self.assertEqual(put2_headers.get('X-Delete-At'), '1577383915')
self.assertEqual(put2_headers.get('X-Object-Meta-Dog'), 'Rantanplan')
self.assertIsNone(put2_headers.get('X-Horse'))
class TestUntar(unittest.TestCase):
@ -610,6 +634,7 @@ class TestUntar(unittest.TestCase):
def test_extract_tar_fail_unicode(self):
dir_tree = [{'sub_dir1': ['sub1_file1']},
{'sub_dir2': [b'sub2\xdefile1', 'sub2_file2']},
{b'good_\xe2\x98\x83': [{'still_good': b'\xe2\x98\x83'}]},
{b'sub_\xdedir3': [{'sub4_dir1': 'sub4_file1'}]}]
self.build_tar(dir_tree)
req = Request.blank('/tar_works/acc/',
@ -619,13 +644,18 @@ class TestUntar(unittest.TestCase):
req.headers['transfer-encoding'] = 'chunked'
resp_body = self.handle_extract_and_iter(req, '')
resp_data = utils.json.loads(resp_body)
self.assertEqual(self.app.calls, 4)
self.assertEqual(resp_data['Number Files Created'], 2)
self.assertEqual(self.app.calls, 6)
self.assertEqual(resp_data['Number Files Created'], 3)
self.assertEqual(resp_data['Response Status'], '400 Bad Request')
self.assertEqual(
resp_data['Errors'],
[['sub_dir2/sub2%DEfile1', '412 Precondition Failed'],
['sub_%DEdir3/sub4_dir1/sub4_file1', '412 Precondition Failed']])
self.assertEqual(self.app.put_paths, [
'/tar_works/acc/sub_dir1/sub1_file1',
'/tar_works/acc/sub_dir2/sub2_file2',
'/tar_works/acc/good_\xe2\x98\x83/still_good/\xe2\x98\x83',
])
def test_get_response_body(self):
txt_body = bulk.get_response_body(
@ -1027,5 +1057,6 @@ class TestSwiftInfo(unittest.TestCase):
swift_info['bulk_delete'].get('max_failed_deletes'),
numbers.Integral))
if __name__ == '__main__':
unittest.main()