openstack/swift
Code Issues Proposed changes

Re-format the SLO manifest file on new multipart-manifest GET call

Browse Source 
Currently, the multipart-manifest=get call returns output in json
format that is inconsistent with the format that is used for the
multipart-manifest=put.

This in turn introduces a new call: ?multipart-manifest=get&format=raw

Change-Id: I2242943a738f667cbda6363bcb6a017f341e834f
Closes-Bug: 1252482
This commit is contained in:
Bill Huber 2016-01-05 14:40:50 -06:00 committed by Alistair Coles
parent 7cc2c783a4
commit e1f1296088
3 changed files with 182 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
swift/common/middleware/slo.py
View File

@ -149,9 +149,15 @@ A GET request with the query parameter::
?multipart-manifest=get
Will return the actual manifest file itself. This is generated json and does
not match the data sent from the original multipart-manifest=put. This call's
main purpose is for debugging.
will return a transformed version of the original manifest, containing
additional fields and different key names.
A GET request with the query parameters::
?multipart-manifest=get&format=raw
will return the contents of the original manifest as it was sent by the client.
The main purpose for both calls is solely debugging.
When the manifest object is uploaded you are more or less guaranteed that
every segment in the manifest exists and matched the specifications.
@ -573,6 +579,9 @@ class SloGetContext(WSGIContext):
# Handle pass-through request for the manifest itself
if req.params.get('multipart-manifest') == 'get':
if req.params.get('format') == 'raw':
resp_iter = self.convert_segment_listing(
self._response_headers, resp_iter)
new_headers = []
for header, value in self._response_headers:
if header.lower() == 'content-type':
@ -606,7 +615,40 @@ class SloGetContext(WSGIContext):
req, resp_headers, resp_iter)
return response(req.environ, start_response)
def get_or_head_response(self, req, resp_headers, resp_iter):
def convert_segment_listing(self, resp_headers, resp_iter):
"""
Converts the manifest data to match with the format
that was put in through ?multipart-manifest=put
:param resp_headers: response headers
:param resp_iter: a response iterable
"""
segments = self._get_manifest_read(resp_iter)
for seg_dict in segments:
seg_dict.pop('content_type', None)
seg_dict.pop('last_modified', None)
seg_dict.pop('sub_slo', None)
seg_dict['path'] = seg_dict.pop('name', None)
seg_dict['size_bytes'] = seg_dict.pop('bytes', None)
seg_dict['etag'] = seg_dict.pop('hash', None)
json_data = json.dumps(segments) # convert to string
if six.PY3:
json_data = json_data.encode('utf-8')
new_headers = []
for header, value in resp_headers:
if header.lower() == 'content-length':
new_headers.append(('Content-Length',
len(json_data)))
else:
new_headers.append((header, value))
self._response_headers = new_headers
return [json_data]
def _get_manifest_read(self, resp_iter):
with closing_if_possible(resp_iter):
resp_body = ''.join(resp_iter)
try:
@ -614,6 +656,11 @@ class SloGetContext(WSGIContext):
except ValueError:
segments = []
return segments
def get_or_head_response(self, req, resp_headers, resp_iter):
segments = self._get_manifest_read(resp_iter)
etag = md5()
content_length = 0
for seg_dict in segments:

33
test/functional/tests.py
View File

@ -3242,6 +3242,39 @@ class TestSlo(Base):
self.assertEqual(value[1]['name'],
'/%s/seg_b' % self.env.container.name.decode("utf-8"))
def test_slo_get_raw_the_manifest_with_details_from_server(self):
manifest = self.env.container.file("manifest-db")
got_body = manifest.read(parms={'multipart-manifest': 'get',
'format': 'raw'})
self.assertEqual('application/json; charset=utf-8',
manifest.content_type)
try:
value = json.loads(got_body)
except ValueError:
msg = "GET with multipart-manifest=get&format=raw got invalid json"
self.fail(msg)
self.assertEqual(
set(value[0].keys()), set(('size_bytes', 'etag', 'path')))
self.assertEqual(len(value), 2)
self.assertEqual(value[0]['size_bytes'], 1024 * 1024)
self.assertEqual(value[0]['etag'],
hashlib.md5('d' * 1024 * 1024).hexdigest())
self.assertEqual(value[0]['path'],
'/%s/seg_d' % self.env.container.name.decode("utf-8"))
self.assertEqual(value[1]['size_bytes'], 1024 * 1024)
self.assertEqual(value[1]['etag'],
hashlib.md5('b' * 1024 * 1024).hexdigest())
self.assertEqual(value[1]['path'],
'/%s/seg_b' % self.env.container.name.decode("utf-8"))
file_item = self.env.container.file("manifest-from-get-raw")
file_item.write(got_body, parms={'multipart-manifest': 'put'})
file_contents = file_item.read()
self.assertEqual(2 * 1024 * 1024, len(file_contents))
def test_slo_head_the_manifest(self):
manifest = self.env.container.file("manifest-abcde")
got_info = manifest.info(parms={'multipart-manifest': 'get'})

99
test/unit/common/middleware/test_slo.py
View File

@ -1072,6 +1072,103 @@ class TestSloHeadManifest(SloTestCase):
self.assertEqual(status, '304 Not Modified')
class TestSloGetRawManifest(SloTestCase):
def setUp(self):
super(TestSloGetRawManifest, self).setUp()
_bc_manifest_json = json.dumps(
[{'name': '/gettest/b_10', 'hash': md5hex('b' * 10), 'bytes': '10',
'content_type': 'text/plain',
'last_modified': '1970-01-01T00:00:00.000000'},
{'name': '/gettest/c_15', 'hash': md5hex('c' * 15), 'bytes': '15',
'content_type': 'text/plain',
'last_modified': '1970-01-01T00:00:00.000000'},
{'name': '/gettest/d_10',
'hash': md5hex(md5hex("e" * 5) + md5hex("f" * 5)), 'bytes': '10',
'content_type': 'application/json;swift_bytes=10',
'sub_slo': True,
'last_modified': '1970-01-01T00:00:00.000000'}])
self.bc_etag = md5hex(_bc_manifest_json)
self.app.register(
'GET', '/v1/AUTH_test/gettest/manifest-bc',
swob.HTTPOk, {'Content-Type': 'application/json;swift_bytes=35',
'X-Static-Large-Object': 'true',
'X-Object-Meta-Plant': 'Ficus',
'Etag': md5hex(_bc_manifest_json)},
_bc_manifest_json)
_bc_manifest_json_ranges = json.dumps(
[{'name': '/gettest/b_10', 'hash': md5hex('b' * 10), 'bytes': '10',
'last_modified': '1970-01-01T00:00:00.000000',
'content_type': 'text/plain', 'range': '1-99'},
{'name': '/gettest/c_15', 'hash': md5hex('c' * 15), 'bytes': '15',
'last_modified': '1970-01-01T00:00:00.000000',
'content_type': 'text/plain', 'range': '100-200'}])
self.app.register(
'GET', '/v1/AUTH_test/gettest/manifest-bc-r',
swob.HTTPOk, {'Content-Type': 'application/json;swift_bytes=25',
'X-Static-Large-Object': 'true',
'X-Object-Meta-Plant': 'Ficus',
'Etag': md5hex(_bc_manifest_json_ranges)},
_bc_manifest_json_ranges)
def test_get_raw_manifest(self):
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-bc'
'?multipart-manifest=get&format=raw',
environ={'REQUEST_METHOD': 'GET',
'HTTP_ACCEPT': 'application/json'})
status, headers, body = self.call_slo(req)
self.assertEqual(status, '200 OK')
self.assertTrue(('Etag', self.bc_etag) in headers, headers)
self.assertTrue(('X-Static-Large-Object', 'true') in headers, headers)
self.assertTrue(
('Content-Type', 'application/json; charset=utf-8') in headers,
headers)
try:
resp_data = json.loads(body)
except ValueError:
self.fail("Invalid JSON in manifest GET: %r" % body)
self.assertEqual(
resp_data,
[{'etag': md5hex('b' * 10), 'size_bytes': '10',
'path': '/gettest/b_10'},
{'etag': md5hex('c' * 15), 'size_bytes': '15',
'path': '/gettest/c_15'},
{'etag': md5hex(md5hex("e" * 5) + md5hex("f" * 5)),
'size_bytes': '10',
'path': '/gettest/d_10'}])
def test_get_raw_manifest_passthrough_with_ranges(self):
req = Request.blank(
'/v1/AUTH_test/gettest/manifest-bc-r'
'?multipart-manifest=get&format=raw',
environ={'REQUEST_METHOD': 'GET',
'HTTP_ACCEPT': 'application/json'})
status, headers, body = self.call_slo(req)
self.assertEqual(status, '200 OK')
self.assertTrue(
('Content-Type', 'application/json; charset=utf-8') in headers,
headers)
try:
resp_data = json.loads(body)
except ValueError:
self.fail("Invalid JSON in manifest GET: %r" % body)
self.assertEqual(
resp_data,
[{'etag': md5hex('b' * 10), 'size_bytes': '10',
'path': '/gettest/b_10', 'range': '1-99'},
{'etag': md5hex('c' * 15), 'size_bytes': '15',
'path': '/gettest/c_15', 'range': '100-200'}],
body)
class TestSloGetManifest(SloTestCase):
def setUp(self):
super(TestSloGetManifest, self).setUp()
@ -1777,7 +1874,7 @@ class TestSloGetManifest(SloTestCase):
self.assertEqual(
body, 'aaaaabbbbbbbbbbcccccccccccccccdddddddddddddddddddd')
def test_get_segment_with_non_ascii_name(self):
def test_get_segment_with_non_ascii_path(self):
segment_body = u"a møøse once bit my sister".encode("utf-8")
self.app.register(
'GET', u'/v1/AUTH_test/ünicode/öbject-segment'.encode('utf-8'),