Merge "Allow DLO manifest to be updated when using post-as-copy"

doc/source/api/large_objects.rst

@@ -168,6 +168,12 @@ of segments to a second location and update the manifest to point to
 this new location. During the upload of the new segments, the original
 manifest is still available to download the first set of segments.
 
+.. note::
+
+  When updating a manifest object using a POST request, a
+  ``X-Object-Manifest`` header must be included for the
+  object to continue to behave as a manifest object.
+
 **Example Upload segment of large object request: HTTP**
 
 .. code::

doc/source/overview_large_objects.rst

@@ -57,6 +57,10 @@ Additional Notes
   <container>/<prefix>`` header will be returned with the concatenated object
   so you can tell where it's getting its segments from.
 
+* When updating a manifest object using a POST request, a
+  ``X-Object-Manifest`` header must be included for the object to
+  continue to behave as a manifest object.
+
 * The response's ``Content-Length`` for a ``GET`` or ``HEAD`` on the manifest
   file will be the sum of all the segments in the ``<container>/<prefix>``
   listing, dynamically. So, uploading additional segments after the manifest is

swift/common/middleware/copy.py

@@ -489,8 +489,9 @@ class ServerSideCopyMiddleware(object):
                 params['multipart-manifest'] = 'put'
             if 'X-Object-Manifest' in source_resp.headers:
                 del params['multipart-manifest']
-                sink_req.headers['X-Object-Manifest'] = \
+                if 'swift.post_as_copy' not in sink_req.environ:
-                    source_resp.headers['X-Object-Manifest']
+                    sink_req.headers['X-Object-Manifest'] = \
+                        source_resp.headers['X-Object-Manifest']
             sink_req.params = params
 
         # Set data source, content length and etag for the PUT request

swift/common/middleware/dlo.py

@@ -85,13 +85,17 @@ available to download the first set of segments.
 
 .. note::
 
+    When updating a manifest object using a POST request, a
+    ``X-Object-Manifest`` header must be included for the object to
+    continue to behave as a manifest object.
+
     The manifest file should have no content. However, this is not enforced.
     If the manifest path itself conforms to container/prefix specified in
-    X-Object-Manifest, and if manifest has some content/data in it, it would
+    ``X-Object-Manifest``, and if manifest has some content/data in it, it
-    also be considered as segment and manifest's content will be part of the
+    would also be considered as segment and manifest's content will be part of
-    concatenated GET response. The order of concatenation follows the usual DLO
+    the concatenated GET response. The order of concatenation follows the usual
-    logic which is - the order of concatenation adheres to order returned when
+    DLO logic which is - the order of concatenation adheres to order returned
-    segment names are sorted.
+    when segment names are sorted.
 
 
 Here's an example using ``curl`` with tiny 1-byte segments::

test/functional/tests.py

@@ -2750,6 +2750,7 @@ class TestDlo(Base):
     def test_dlo_post_with_manifest_header(self):
         # verify that performing a POST to a DLO manifest
         # preserves the fact that it is a manifest file.
+        # verify that the x-object-manifest header may be updated.
 
         # create a new manifest for this test to avoid test coupling.
         x_o_m = self.env.container.file('man1').info()['x_object_manifest']
@@ -2763,25 +2764,105 @@ class TestDlo(Base):
         contents = file_item.read(parms={})
         self.assertEqual(expected_contents, contents)
 
-        # POST to the manifest file
+        # POST a modified x-object-manifest value
-        # include the x-object-manifest in case running with fast-post
+        new_x_o_m = x_o_m.rstrip('lower') + 'upper'
         file_item.post({'x-object-meta-foo': 'bar',
-                        'x-object-manifest': x_o_m})
+                        'x-object-manifest': new_x_o_m})
 
-        # Verify x-object-manifest still intact
+        # verify that x-object-manifest was updated
         file_item.info()
         resp_headers = file_item.conn.response.getheaders()
-        self.assertIn(('x-object-manifest', x_o_m), resp_headers)
+        self.assertIn(('x-object-manifest', new_x_o_m), resp_headers)
         self.assertIn(('x-object-meta-foo', 'bar'), resp_headers)
 
         # verify that manifest content was not changed
         manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
         self.assertEqual('manifest-contents', manifest_contents)
 
-        # verify that manifest still points to original content
+        # verify that updated manifest points to new content
+        expected_contents = ''.join([(c * 10) for c in 'ABCDE'])
         contents = file_item.read(parms={})
         self.assertEqual(expected_contents, contents)
 
+        # Now revert the manifest to point to original segments, including a
+        # multipart-manifest=get param just to check that has no effect
+        file_item.post({'x-object-manifest': x_o_m},
+                       parms={'multipart-manifest': 'get'})
+
+        # verify that x-object-manifest was reverted
+        info = file_item.info()
+        self.assertIn('x_object_manifest', info)
+        self.assertEqual(x_o_m, info['x_object_manifest'])
+
+        # verify that manifest content was not changed
+        manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
+        self.assertEqual('manifest-contents', manifest_contents)
+
+        # verify that updated manifest points new content
+        expected_contents = ''.join([(c * 10) for c in 'abcde'])
+        contents = file_item.read(parms={})
+        self.assertEqual(expected_contents, contents)
+
+    def test_dlo_post_without_manifest_header(self):
+        # verify that a POST to a DLO manifest object with no
+        # x-object-manifest header will cause the existing x-object-manifest
+        # header to be lost
+
+        # create a new manifest for this test to avoid test coupling.
+        x_o_m = self.env.container.file('man1').info()['x_object_manifest']
+        file_item = self.env.container.file(Utils.create_name())
+        file_item.write('manifest-contents', hdrs={"X-Object-Manifest": x_o_m})
+
+        # sanity checks
+        manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
+        self.assertEqual('manifest-contents', manifest_contents)
+        expected_contents = ''.join([(c * 10) for c in 'abcde'])
+        contents = file_item.read(parms={})
+        self.assertEqual(expected_contents, contents)
+
+        # POST with no x-object-manifest header
+        file_item.post({})
+
+        # verify that existing x-object-manifest was removed
+        info = file_item.info()
+        self.assertNotIn('x_object_manifest', info)
+
+        # verify that object content was not changed
+        manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
+        self.assertEqual('manifest-contents', manifest_contents)
+
+        # verify that object is no longer a manifest
+        contents = file_item.read(parms={})
+        self.assertEqual('manifest-contents', contents)
+
+    def test_dlo_post_with_manifest_regular_object(self):
+        # verify that performing a POST to a regular object
+        # with a manifest header will create a DLO.
+
+        # Put a regular object
+        file_item = self.env.container.file(Utils.create_name())
+        file_item.write('file contents', hdrs={})
+
+        # sanity checks
+        file_contents = file_item.read(parms={})
+        self.assertEqual('file contents', file_contents)
+
+        # get the path associated with man1
+        x_o_m = self.env.container.file('man1').info()['x_object_manifest']
+
+        # POST a x-object-manifest value to the regular object
+        file_item.post({'x-object-manifest': x_o_m})
+
+        # verify that the file is now a manifest
+        manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
+        self.assertEqual('file contents', manifest_contents)
+        expected_contents = ''.join([(c * 10) for c in 'abcde'])
+        contents = file_item.read(parms={})
+        self.assertEqual(expected_contents, contents)
+        file_item.info()
+        resp_headers = file_item.conn.response.getheaders()
+        self.assertIn(('x-object-manifest', x_o_m), resp_headers)
+
 
 class TestDloUTF8(Base2, TestDlo):
     set_up = False

test/unit/common/middleware/test_copy.py

@@ -210,6 +210,38 @@ class TestServerSideCopyMiddleware(unittest.TestCase):
         self.assertEqual(len(self.authorized), 1)
         self.assertRequestEqual(req, self.authorized[0])
 
+    def test_POST_as_COPY_dynamic_large_object_manifest(self):
+        self.app.register('GET', '/v1/a/c/o', swob.HTTPOk,
+                          {'X-Object-Manifest': 'orig_manifest'}, 'passed')
+        self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {})
+        req = Request.blank('/v1/a/c/o', method='POST',
+                            headers={'X-Object-Manifest': 'new_manifest'})
+        status, headers, body = self.call_ssc(req)
+        self.assertEqual(status, '202 Accepted')
+
+        calls = self.app.calls_with_headers
+        method, path, req_headers = calls[1]
+        self.assertEqual('PUT', method)
+        self.assertEqual('new_manifest', req_headers['x-object-manifest'])
+        self.assertEqual(len(self.authorized), 1)
+        self.assertRequestEqual(req, self.authorized[0])
+
+    def test_POST_as_COPY_dynamic_large_object_no_manifest(self):
+        self.app.register('GET', '/v1/a/c/o', swob.HTTPOk,
+                          {'X-Object-Manifest': 'orig_manifest'}, 'passed')
+        self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {})
+        req = Request.blank('/v1/a/c/o', method='POST',
+                            headers={})
+        status, headers, body = self.call_ssc(req)
+        self.assertEqual(status, '202 Accepted')
+
+        calls = self.app.calls_with_headers
+        method, path, req_headers = calls[1]
+        self.assertEqual('PUT', method)
+        self.assertNotIn('X-Object-Manifest', req_headers)
+        self.assertEqual(len(self.authorized), 1)
+        self.assertRequestEqual(req, self.authorized[0])
+
     def test_basic_put_with_x_copy_from(self):
         self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
         self.app.register('PUT', '/v1/a/c/o2', swob.HTTPCreated, {})