Allow bulk to fwd some headers at tar extraction

Whitelisted headers include X-Delete-At/X-Delete-After and all Object
metadata headers (X-Object-Meta-*)

Closes-Bug: 1857546
Change-Id: If5fb164693e395f89d57899fb8ab355f1e3f817c

swift/common/middleware/bulk.py

@@ -98,6 +98,10 @@ The bulk middleware  will handle xattrs stored by both GNU and BSD tar (2).
 Only xattrs ``user.mime_type`` and ``user.meta.*`` are processed. Other
 attributes are ignored.
 
+In addition to the extended attributes, the object metadata and the
+x-delete-at/x-delete-after headers set in the request are also assigned to the
+extracted objects.
+
 Notes:
 
 (1) The POSIX 1003.1-2001 (pax) format. The default format on GNU tar
@@ -206,6 +210,7 @@ from swift.common.utils import get_logger, register_swift_info, \
     StreamingPile
 from swift.common import constraints
 from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND, HTTP_CONFLICT
+from swift.common.request_helpers import is_user_meta
 from swift.common.wsgi import make_subrequest
 
 
@@ -622,6 +627,12 @@ class Bulk(object):
                         'X-Auth-Token': req.headers.get('X-Auth-Token'),
                     }
 
+                    # Copy some whitelisted headers to the subrequest
+                    for k, v in req.headers.items():
+                        if ((k.lower() in ('x-delete-at', 'x-delete-after'))
+                                or is_user_meta('object', k)):
+                            create_headers[k] = v
+
                     create_obj_req = make_subrequest(
                         req.environ, method='PUT',
                         path=wsgi_quote(destination),

test/unit/common/middleware/test_bulk.py

@@ -41,12 +41,21 @@ class FakeApp(object):
     def __init__(self):
         self.calls = 0
         self.delete_paths = []
+        self.put_paths = []
         self.max_pathlen = 100
         self.del_cont_total_calls = 2
         self.del_cont_cur_call = 0
 
     def __call__(self, env, start_response):
         self.calls += 1
+        if env.get('swift.source') in ('EA', 'BD'):
+            assert not env.get('swift.proxy_access_log_made')
+        if not six.PY2:
+            # Check that it's valid WSGI
+            assert all(0 <= ord(c) <= 255 for c in env['PATH_INFO'])
+
+        if env['REQUEST_METHOD'] == 'PUT':
+            self.put_paths.append(env['PATH_INFO'])
         if env['PATH_INFO'].startswith('/unauth/'):
             if env['PATH_INFO'].endswith('/c/f_ok'):
                 return Response(status='204 No Content')(env, start_response)
@@ -224,8 +233,16 @@ class TestUntarMetadata(unittest.TestCase):
         req = Request.blank('/v1/a/c?extract-archive=tar')
         req.environ['REQUEST_METHOD'] = 'PUT'
         req.environ['wsgi.input'] = tar_ball
-        req.headers['transfer-encoding'] = 'chunked'
+        # Since there should be a proxy-logging left of us...
-        req.headers['accept'] = 'application/json;q=1.0'
+        req.environ['swift.proxy_access_log_made'] = True
+        req.headers.update({
+            'transfer-encoding': 'chunked',
+            'accept': 'application/json;q=1.0',
+            'X-Delete-At': '1577383915',
+            'X-Object-Meta-Dog': 'Rantanplan',
+            'X-Horse': 'Jolly Jumper',
+            'X-Object-Meta-Cat': 'tabby',
+        })
 
         resp = req.get_response(self.bulk)
         self.assertEqual(resp.status_int, 200)
@@ -244,12 +261,19 @@ class TestUntarMetadata(unittest.TestCase):
         self.assertEqual(
             put1_headers.get('X-Object-Meta-Afternoon-Snack'),
             'gigantic bucket of coffee')
+        self.assertEqual(put1_headers.get('X-Delete-At'), '1577383915')
+        self.assertEqual(put1_headers.get('X-Object-Meta-Dog'), 'Rantanplan')
+        self.assertEqual(put1_headers.get('X-Object-Meta-Cat'), 'tabby')
+        self.assertIsNone(put1_headers.get('X-Horse'))
 
         put2_headers = HeaderKeyDict(self.app.calls_with_headers[2][2])
         self.assertEqual(put2_headers.get('X-Object-Meta-Muppet'), 'bert')
         self.assertEqual(put2_headers.get('X-Object-Meta-Cat'), 'fluffy')
         self.assertIsNone(put2_headers.get('Content-Type'))
         self.assertIsNone(put2_headers.get('X-Object-Meta-Blah'))
+        self.assertEqual(put2_headers.get('X-Delete-At'), '1577383915')
+        self.assertEqual(put2_headers.get('X-Object-Meta-Dog'), 'Rantanplan')
+        self.assertIsNone(put2_headers.get('X-Horse'))
 
 
 class TestUntar(unittest.TestCase):
@@ -610,6 +634,7 @@ class TestUntar(unittest.TestCase):
     def test_extract_tar_fail_unicode(self):
         dir_tree = [{'sub_dir1': ['sub1_file1']},
                     {'sub_dir2': [b'sub2\xdefile1', 'sub2_file2']},
+                    {b'good_\xe2\x98\x83': [{'still_good': b'\xe2\x98\x83'}]},
                     {b'sub_\xdedir3': [{'sub4_dir1': 'sub4_file1'}]}]
         self.build_tar(dir_tree)
         req = Request.blank('/tar_works/acc/',
@@ -619,13 +644,18 @@ class TestUntar(unittest.TestCase):
         req.headers['transfer-encoding'] = 'chunked'
         resp_body = self.handle_extract_and_iter(req, '')
         resp_data = utils.json.loads(resp_body)
-        self.assertEqual(self.app.calls, 4)
+        self.assertEqual(self.app.calls, 6)
-        self.assertEqual(resp_data['Number Files Created'], 2)
+        self.assertEqual(resp_data['Number Files Created'], 3)
         self.assertEqual(resp_data['Response Status'], '400 Bad Request')
         self.assertEqual(
             resp_data['Errors'],
             [['sub_dir2/sub2%DEfile1', '412 Precondition Failed'],
              ['sub_%DEdir3/sub4_dir1/sub4_file1', '412 Precondition Failed']])
+        self.assertEqual(self.app.put_paths, [
+            '/tar_works/acc/sub_dir1/sub1_file1',
+            '/tar_works/acc/sub_dir2/sub2_file2',
+            '/tar_works/acc/good_\xe2\x98\x83/still_good/\xe2\x98\x83',
+        ])
 
     def test_get_response_body(self):
         txt_body = bulk.get_response_body(
@@ -1027,5 +1057,6 @@ class TestSwiftInfo(unittest.TestCase):
             swift_info['bulk_delete'].get('max_failed_deletes'),
             numbers.Integral))
 
+
 if __name__ == '__main__':
     unittest.main()