openstack/swift
Code Issues Proposed changes

Merge "Allow DLO manifest to be updated when using post-as-copy"

Browse Source
This commit is contained in:
Jenkins 2016-09-08 15:09:27 +00:00 committed by Gerrit Code Review
commit 7b9e2d27ea
6 changed files with 141 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
doc/source/api/large_objects.rst
View File

@ -168,6 +168,12 @@ of segments to a second location and update the manifest to point to
this new location. During the upload of the new segments, the original
manifest is still available to download the first set of segments.
.. note::
When updating a manifest object using a POST request, a
``X-Object-Manifest`` header must be included for the
object to continue to behave as a manifest object.
**Example Upload segment of large object request: HTTP**
.. code::

4
doc/source/overview_large_objects.rst
View File

@ -57,6 +57,10 @@ Additional Notes
<container>/<prefix>`` header will be returned with the concatenated object
so you can tell where it's getting its segments from.
* When updating a manifest object using a POST request, a
``X-Object-Manifest`` header must be included for the object to
continue to behave as a manifest object.
* The response's ``Content-Length`` for a ``GET`` or ``HEAD`` on the manifest
file will be the sum of all the segments in the ``<container>/<prefix>``
listing, dynamically. So, uploading additional segments after the manifest is

5
swift/common/middleware/copy.py
View File

@ -489,8 +489,9 @@ class ServerSideCopyMiddleware(object):
params['multipart-manifest'] = 'put'
if 'X-Object-Manifest' in source_resp.headers:
del params['multipart-manifest']
sink_req.headers['X-Object-Manifest'] = \
source_resp.headers['X-Object-Manifest']
if 'swift.post_as_copy' not in sink_req.environ:
sink_req.headers['X-Object-Manifest'] = \
source_resp.headers['X-Object-Manifest']
sink_req.params = params
# Set data source, content length and etag for the PUT request

14
swift/common/middleware/dlo.py
View File

@ -85,13 +85,17 @@ available to download the first set of segments.
.. note::
When updating a manifest object using a POST request, a
``X-Object-Manifest`` header must be included for the object to
continue to behave as a manifest object.
The manifest file should have no content. However, this is not enforced.
If the manifest path itself conforms to container/prefix specified in
X-Object-Manifest, and if manifest has some content/data in it, it would
also be considered as segment and manifest's content will be part of the
concatenated GET response. The order of concatenation follows the usual DLO
logic which is - the order of concatenation adheres to order returned when
segment names are sorted.
``X-Object-Manifest``, and if manifest has some content/data in it, it
would also be considered as segment and manifest's content will be part of
the concatenated GET response. The order of concatenation follows the usual
DLO logic which is - the order of concatenation adheres to order returned
when segment names are sorted.
Here's an example using ``curl`` with tiny 1-byte segments::

93
test/functional/tests.py
View File

@ -2750,6 +2750,7 @@ class TestDlo(Base):
def test_dlo_post_with_manifest_header(self):
# verify that performing a POST to a DLO manifest
# preserves the fact that it is a manifest file.
# verify that the x-object-manifest header may be updated.
# create a new manifest for this test to avoid test coupling.
x_o_m = self.env.container.file('man1').info()['x_object_manifest']
@ -2763,25 +2764,105 @@ class TestDlo(Base):
contents = file_item.read(parms={})
self.assertEqual(expected_contents, contents)
# POST to the manifest file
# include the x-object-manifest in case running with fast-post
# POST a modified x-object-manifest value
new_x_o_m = x_o_m.rstrip('lower') + 'upper'
file_item.post({'x-object-meta-foo': 'bar',
'x-object-manifest': x_o_m})
'x-object-manifest': new_x_o_m})
# Verify x-object-manifest still intact
# verify that x-object-manifest was updated
file_item.info()
resp_headers = file_item.conn.response.getheaders()
self.assertIn(('x-object-manifest', x_o_m), resp_headers)
self.assertIn(('x-object-manifest', new_x_o_m), resp_headers)
self.assertIn(('x-object-meta-foo', 'bar'), resp_headers)
# verify that manifest content was not changed
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
self.assertEqual('manifest-contents', manifest_contents)
# verify that manifest still points to original content
# verify that updated manifest points to new content
expected_contents = ''.join([(c * 10) for c in 'ABCDE'])
contents = file_item.read(parms={})
self.assertEqual(expected_contents, contents)
# Now revert the manifest to point to original segments, including a
# multipart-manifest=get param just to check that has no effect
file_item.post({'x-object-manifest': x_o_m},
parms={'multipart-manifest': 'get'})
# verify that x-object-manifest was reverted
info = file_item.info()
self.assertIn('x_object_manifest', info)
self.assertEqual(x_o_m, info['x_object_manifest'])
# verify that manifest content was not changed
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
self.assertEqual('manifest-contents', manifest_contents)
# verify that updated manifest points new content
expected_contents = ''.join([(c * 10) for c in 'abcde'])
contents = file_item.read(parms={})
self.assertEqual(expected_contents, contents)
def test_dlo_post_without_manifest_header(self):
# verify that a POST to a DLO manifest object with no
# x-object-manifest header will cause the existing x-object-manifest
# header to be lost
# create a new manifest for this test to avoid test coupling.
x_o_m = self.env.container.file('man1').info()['x_object_manifest']
file_item = self.env.container.file(Utils.create_name())
file_item.write('manifest-contents', hdrs={"X-Object-Manifest": x_o_m})
# sanity checks
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
self.assertEqual('manifest-contents', manifest_contents)
expected_contents = ''.join([(c * 10) for c in 'abcde'])
contents = file_item.read(parms={})
self.assertEqual(expected_contents, contents)
# POST with no x-object-manifest header
file_item.post({})
# verify that existing x-object-manifest was removed
info = file_item.info()
self.assertNotIn('x_object_manifest', info)
# verify that object content was not changed
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
self.assertEqual('manifest-contents', manifest_contents)
# verify that object is no longer a manifest
contents = file_item.read(parms={})
self.assertEqual('manifest-contents', contents)
def test_dlo_post_with_manifest_regular_object(self):
# verify that performing a POST to a regular object
# with a manifest header will create a DLO.
# Put a regular object
file_item = self.env.container.file(Utils.create_name())
file_item.write('file contents', hdrs={})
# sanity checks
file_contents = file_item.read(parms={})
self.assertEqual('file contents', file_contents)
# get the path associated with man1
x_o_m = self.env.container.file('man1').info()['x_object_manifest']
# POST a x-object-manifest value to the regular object
file_item.post({'x-object-manifest': x_o_m})
# verify that the file is now a manifest
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
self.assertEqual('file contents', manifest_contents)
expected_contents = ''.join([(c * 10) for c in 'abcde'])
contents = file_item.read(parms={})
self.assertEqual(expected_contents, contents)
file_item.info()
resp_headers = file_item.conn.response.getheaders()
self.assertIn(('x-object-manifest', x_o_m), resp_headers)
class TestDloUTF8(Base2, TestDlo):
set_up = False

32
test/unit/common/middleware/test_copy.py
View File

@ -210,6 +210,38 @@ class TestServerSideCopyMiddleware(unittest.TestCase):
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_POST_as_COPY_dynamic_large_object_manifest(self):
self.app.register('GET', '/v1/a/c/o', swob.HTTPOk,
{'X-Object-Manifest': 'orig_manifest'}, 'passed')
self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/o', method='POST',
headers={'X-Object-Manifest': 'new_manifest'})
status, headers, body = self.call_ssc(req)
self.assertEqual(status, '202 Accepted')
calls = self.app.calls_with_headers
method, path, req_headers = calls[1]
self.assertEqual('PUT', method)
self.assertEqual('new_manifest', req_headers['x-object-manifest'])
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_POST_as_COPY_dynamic_large_object_no_manifest(self):
self.app.register('GET', '/v1/a/c/o', swob.HTTPOk,
{'X-Object-Manifest': 'orig_manifest'}, 'passed')
self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {})
req = Request.blank('/v1/a/c/o', method='POST',
headers={})
status, headers, body = self.call_ssc(req)
self.assertEqual(status, '202 Accepted')
calls = self.app.calls_with_headers
method, path, req_headers = calls[1]
self.assertEqual('PUT', method)
self.assertNotIn('X-Object-Manifest', req_headers)
self.assertEqual(len(self.authorized), 1)
self.assertRequestEqual(req, self.authorized[0])
def test_basic_put_with_x_copy_from(self):
self.app.register('GET', '/v1/a/c/o', swob.HTTPOk, {}, 'passed')
self.app.register('PUT', '/v1/a/c/o2', swob.HTTPCreated, {})