s3api: Add basic support for ?versions bucket listings
We still don't have support for toggling S3 bucket versioning, but we can at least support getting the latest versions of all objects. See https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGETVersion.html for more information about the API. Note that the returned format is distinct from both "GET Bucket (List Objects) Version 1" and "GET Bucket (List Objects) Version 2" APIs. Change-Id: Ic57c273a3d5d7cdc34ca3a03e35e99b202a0bb01
This commit is contained in:
karen chan
Tim Burke
parent
16fe18ae3b
commit
a2fb335e4b
@ -110,17 +110,22 @@ class BucketController(Controller):
|
||||
'format': 'json',
|
||||
'limit': max_keys + 1,
|
||||
}
|
||||
if 'marker' in req.params:
|
||||
query.update({'marker': req.params['marker']})
|
||||
if 'prefix' in req.params:
|
||||
query.update({'prefix': req.params['prefix']})
|
||||
if 'delimiter' in req.params:
|
||||
query.update({'delimiter': req.params['delimiter']})
|
||||
|
||||
# GET Bucket (List Objects) Version 2 parameters
|
||||
is_v2 = int(req.params.get('list-type', '1')) == 2
|
||||
fetch_owner = False
|
||||
if is_v2:
|
||||
if 'versions' in req.params:
|
||||
listing_type = 'object-versions'
|
||||
if 'key-marker' in req.params:
|
||||
query.update({'marker': req.params['key-marker']})
|
||||
elif 'version-id-marker' in req.params:
|
||||
err_msg = ('A version-id marker cannot be specified without '
|
||||
'a key marker.')
|
||||
raise InvalidArgument('version-id-marker',
|
||||
req.params['version-id-marker'], err_msg)
|
||||
elif int(req.params.get('list-type', '1')) == 2:
|
||||
listing_type = 'version-2'
|
||||
if 'start-after' in req.params:
|
||||
query.update({'marker': req.params['start-after']})
|
||||
# continuation-token overrides start-after
|
||||
@ -129,44 +134,63 @@ class BucketController(Controller):
|
||||
query.update({'marker': decoded})
|
||||
if 'fetch-owner' in req.params:
|
||||
fetch_owner = config_true_value(req.params['fetch-owner'])
|
||||
else:
|
||||
listing_type = 'version-1'
|
||||
if 'marker' in req.params:
|
||||
query.update({'marker': req.params['marker']})
|
||||
|
||||
resp = req.get_response(self.app, query=query)
|
||||
|
||||
objects = json.loads(resp.body)
|
||||
|
||||
elem = Element('ListBucketResult')
|
||||
SubElement(elem, 'Name').text = req.container_name
|
||||
SubElement(elem, 'Prefix').text = req.params.get('prefix')
|
||||
|
||||
# in order to judge that truncated is valid, check whether
|
||||
# max_keys + 1 th element exists in swift.
|
||||
is_truncated = max_keys > 0 and len(objects) > max_keys
|
||||
objects = objects[:max_keys]
|
||||
|
||||
if not is_v2:
|
||||
SubElement(elem, 'Marker').text = req.params.get('marker')
|
||||
if is_truncated and 'delimiter' in req.params:
|
||||
if 'name' in objects[-1]:
|
||||
SubElement(elem, 'NextMarker').text = \
|
||||
objects[-1]['name']
|
||||
if 'subdir' in objects[-1]:
|
||||
SubElement(elem, 'NextMarker').text = \
|
||||
objects[-1]['subdir']
|
||||
else:
|
||||
if listing_type == 'object-versions':
|
||||
elem = Element('ListVersionsResult')
|
||||
SubElement(elem, 'Name').text = req.container_name
|
||||
SubElement(elem, 'Prefix').text = req.params.get('prefix')
|
||||
SubElement(elem, 'KeyMarker').text = req.params.get('key-marker')
|
||||
SubElement(elem, 'VersionIdMarker').text = req.params.get(
|
||||
'version-id-marker')
|
||||
if is_truncated:
|
||||
if 'name' in objects[-1]:
|
||||
SubElement(elem, 'NextContinuationToken').text = \
|
||||
b64encode(objects[-1]['name'])
|
||||
SubElement(elem, 'NextKeyMarker').text = \
|
||||
objects[-1]['name']
|
||||
if 'subdir' in objects[-1]:
|
||||
SubElement(elem, 'NextContinuationToken').text = \
|
||||
b64encode(objects[-1]['subdir'])
|
||||
if 'continuation-token' in req.params:
|
||||
SubElement(elem, 'ContinuationToken').text = \
|
||||
req.params['continuation-token']
|
||||
if 'start-after' in req.params:
|
||||
SubElement(elem, 'StartAfter').text = \
|
||||
req.params['start-after']
|
||||
SubElement(elem, 'KeyCount').text = str(len(objects))
|
||||
SubElement(elem, 'NextKeyMarker').text = \
|
||||
objects[-1]['subdir']
|
||||
SubElement(elem, 'NextVersionIdMarker').text = 'null'
|
||||
else:
|
||||
elem = Element('ListBucketResult')
|
||||
SubElement(elem, 'Name').text = req.container_name
|
||||
SubElement(elem, 'Prefix').text = req.params.get('prefix')
|
||||
if listing_type == 'version-1':
|
||||
SubElement(elem, 'Marker').text = req.params.get('marker')
|
||||
if is_truncated and 'delimiter' in req.params:
|
||||
if 'name' in objects[-1]:
|
||||
SubElement(elem, 'NextMarker').text = \
|
||||
objects[-1]['name']
|
||||
if 'subdir' in objects[-1]:
|
||||
SubElement(elem, 'NextMarker').text = \
|
||||
objects[-1]['subdir']
|
||||
elif listing_type == 'version-2':
|
||||
if is_truncated:
|
||||
if 'name' in objects[-1]:
|
||||
SubElement(elem, 'NextContinuationToken').text = \
|
||||
b64encode(objects[-1]['name'])
|
||||
if 'subdir' in objects[-1]:
|
||||
SubElement(elem, 'NextContinuationToken').text = \
|
||||
b64encode(objects[-1]['subdir'])
|
||||
if 'continuation-token' in req.params:
|
||||
SubElement(elem, 'ContinuationToken').text = \
|
||||
req.params['continuation-token']
|
||||
if 'start-after' in req.params:
|
||||
SubElement(elem, 'StartAfter').text = \
|
||||
req.params['start-after']
|
||||
SubElement(elem, 'KeyCount').text = str(len(objects))
|
||||
|
||||
SubElement(elem, 'MaxKeys').text = str(tag_max_keys)
|
||||
|
||||
@ -181,8 +205,14 @@ class BucketController(Controller):
|
||||
|
||||
for o in objects:
|
||||
if 'subdir' not in o:
|
||||
contents = SubElement(elem, 'Contents')
|
||||
SubElement(contents, 'Key').text = o['name']
|
||||
if listing_type == 'object-versions':
|
||||
contents = SubElement(elem, 'Version')
|
||||
SubElement(contents, 'Key').text = o['name']
|
||||
SubElement(contents, 'VersionId').text = 'null'
|
||||
SubElement(contents, 'IsLatest').text = 'true'
|
||||
else:
|
||||
contents = SubElement(elem, 'Contents')
|
||||
SubElement(contents, 'Key').text = o['name']
|
||||
SubElement(contents, 'LastModified').text = \
|
||||
o['last_modified'][:-3] + 'Z'
|
||||
if 's3_etag' in o:
|
||||
@ -192,7 +222,7 @@ class BucketController(Controller):
|
||||
etag = '"%s"' % o['hash']
|
||||
SubElement(contents, 'ETag').text = etag
|
||||
SubElement(contents, 'Size').text = str(o['bytes'])
|
||||
if fetch_owner or not is_v2:
|
||||
if fetch_owner or listing_type != 'version-2':
|
||||
owner = SubElement(contents, 'Owner')
|
||||
SubElement(owner, 'ID').text = req.user_id
|
||||
SubElement(owner, 'DisplayName').text = req.user_id
|
||||
|
||||
@ -33,9 +33,9 @@ from test.unit.common.middleware.s3api.helpers import UnreadableInput
|
||||
|
||||
class TestS3ApiBucket(S3ApiTestCase):
|
||||
def setup_objects(self):
|
||||
self.objects = (('rose', '2011-01-05T02:19:14.275290', 0, 303),
|
||||
self.objects = (('lily', '2011-01-05T02:19:14.275290', '0', '3909'),
|
||||
('rose', '2011-01-05T02:19:14.275290', 0, 303),
|
||||
('viola', '2011-01-05T02:19:14.275290', '0', 3909),
|
||||
('lily', '2011-01-05T02:19:14.275290', '0', '3909'),
|
||||
('mu', '2011-01-05T02:19:14.275290',
|
||||
'md5-of-the-manifest; s3_etag=0', '3909'),
|
||||
('with space', '2011-01-05T02:19:14.275290', 0, 390),
|
||||
@ -394,7 +394,7 @@ class TestS3ApiBucket(S3ApiTestCase):
|
||||
status, headers, body = self.call_s3api(req)
|
||||
self.assertEqual(status.split()[0], '200')
|
||||
elem = fromstring(body, 'ListBucketResult')
|
||||
self.assertEqual(elem.find('./NextMarker').text, 'viola')
|
||||
self.assertEqual(elem.find('./NextMarker').text, 'rose')
|
||||
self.assertEqual(elem.find('./MaxKeys').text, '2')
|
||||
self.assertEqual(elem.find('./IsTruncated').text, 'true')
|
||||
|
||||
@ -470,6 +470,46 @@ class TestS3ApiBucket(S3ApiTestCase):
|
||||
for o in objects:
|
||||
self.assertIsNotNone(o.find('./Owner'))
|
||||
|
||||
def test_bucket_GET_with_versions_versioning_not_configured(self):
|
||||
req = Request.blank('/junk?versions',
|
||||
environ={'REQUEST_METHOD': 'GET'},
|
||||
headers={'Authorization': 'AWS test:tester:hmac',
|
||||
'Date': self.get_date_header()})
|
||||
status, headers, body = self.call_s3api(req)
|
||||
|
||||
self.assertEqual(status.split()[0], '200')
|
||||
elem = fromstring(body, 'ListVersionsResult')
|
||||
self.assertEqual(elem.find('./Name').text, 'junk')
|
||||
self.assertIsNone(elem.find('./Prefix').text)
|
||||
self.assertIsNone(elem.find('./KeyMarker').text)
|
||||
self.assertIsNone(elem.find('./VersionIdMarker').text)
|
||||
self.assertEqual(elem.find('./MaxKeys').text, '1000')
|
||||
self.assertEqual(elem.find('./IsTruncated').text, 'false')
|
||||
self.assertEqual(elem.findall('./DeleteMarker'), [])
|
||||
versions = elem.findall('./Version')
|
||||
objects = list(self.objects)
|
||||
self.assertEqual([v.find('./Key').text for v in versions],
|
||||
[v[0] for v in objects])
|
||||
self.assertEqual([v.find('./IsLatest').text for v in versions],
|
||||
['true' for v in objects])
|
||||
self.assertEqual([v.find('./VersionId').text for v in versions],
|
||||
['null' for v in objects])
|
||||
# Last modified in self.objects is 2011-01-05T02:19:14.275290 but
|
||||
# the returned value is 2011-01-05T02:19:14.275Z
|
||||
self.assertEqual([v.find('./LastModified').text for v in versions],
|
||||
[v[1][:-3] + 'Z' for v in objects])
|
||||
self.assertEqual([v.find('./ETag').text for v in versions],
|
||||
['"0"' for v in objects])
|
||||
self.assertEqual([v.find('./Size').text for v in versions],
|
||||
[str(v[3]) for v in objects])
|
||||
self.assertEqual([v.find('./Owner/ID').text for v in versions],
|
||||
['test:tester' for v in objects])
|
||||
self.assertEqual([v.find('./Owner/DisplayName').text
|
||||
for v in versions],
|
||||
['test:tester' for v in objects])
|
||||
self.assertEqual([v.find('./StorageClass').text for v in versions],
|
||||
['STANDARD' for v in objects])
|
||||
|
||||
@s3acl
|
||||
def test_bucket_PUT_error(self):
|
||||
code = self._test_method_error('PUT', '/bucket', swob.HTTPCreated,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user