openstack/swift
Code Issues Proposed changes

s3api: Support GET/HEAD request with ?partNumber

Browse Source 
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Co-Authored-By: Clay Gerrard <clay.gerrard@gmail.com>
Closes-Bug: #1735284
Change-Id: Ib396309c706fbc6bc419377fe23fcf5603a89f45
This commit is contained in:
indianwhocodes 2023-09-11 14:03:01 -07:00 committed by Alistair Coles
parent 6adbeb4036
commit 46e7da97c6
14 changed files with 1569 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
swift/common/middleware/s3api/acl_handlers.py
View File

@ -133,6 +133,15 @@ class BaseAclHandler(object):
query = {}
else:
query = {'version-id': version_id}
if self.req.method == 'HEAD':
# This HEAD for ACL is going to also be the definitive response
# to the client so we need to include client params. We don't
# do this for other client request methods because they may
# have invalid combinations of params and headers for a swift
# HEAD request.
part_number = self.req.params.get('partNumber')
if part_number is not None:
query['part-number'] = part_number
resp = self.req.get_acl_response(app, 'HEAD',
container, obj,
headers, query=query)

10
swift/common/middleware/s3api/controllers/multi_upload.py
View File

@ -196,15 +196,7 @@ class PartController(Controller):
raise InvalidArgument('ResourceType', 'partNumber',
'Unexpected query string parameter')
try:
part_number = int(get_param(req, 'partNumber'))
if part_number < 1 or self.conf.max_upload_part_num < part_number:
raise Exception()
except Exception:
err_msg = 'Part number must be an integer between 1 and %d,' \
' inclusive' % self.conf.max_upload_part_num
raise InvalidArgument('partNumber', get_param(req, 'partNumber'),
err_msg)
part_number = req.validate_part_number()
upload_id = get_param(req, 'uploadId')
_get_upload_info(req, self.app, upload_id)

21
swift/common/middleware/s3api/controllers/obj.py
View File

@ -90,8 +90,14 @@ class ObjectController(Controller):
if version_id not in ('null', None) and \
'object_versioning' not in get_swift_info():
raise S3NotImplemented()
part_number = req.validate_part_number(check_max=False)
query = {}
if version_id is not None:
query['version-id'] = version_id
if part_number is not None:
query['part-number'] = part_number
query = {} if version_id is None else {'version-id': version_id}
if version_id not in ('null', None):
container_info = req.get_container_info(self.app)
if not container_info.get(
@ -101,6 +107,19 @@ class ObjectController(Controller):
resp = req.get_response(self.app, query=query)
if not resp.is_slo:
# SLO ignores part_number for non-slo objects, but s3api only
# allows the query param for non-MPU if it's exactly 1.
part_number = req.validate_part_number(parts_count=1)
if part_number == 1:
# When the query param *is* exactly 1 the response status code
# and headers are updated.
resp.status = HTTP_PARTIAL_CONTENT
resp.headers['Content-Range'] = \
'bytes 0-%d/%s' % (int(resp.headers['Content-Length']) - 1,
resp.headers['Content-Length'])
# else: part_number is None
if req.method == 'HEAD':
resp.app_iter = None

64
swift/common/middleware/s3api/s3request.py
View File

@ -56,7 +56,8 @@ from swift.common.middleware.s3api.s3response import AccessDenied, \
MissingContentLength, InvalidStorageClass, S3NotImplemented, InvalidURI, \
MalformedXML, InvalidRequest, RequestTimeout, InvalidBucketName, \
BadDigest, AuthorizationHeaderMalformed, SlowDown, \
AuthorizationQueryParametersError, ServiceUnavailable, BrokenMPU
AuthorizationQueryParametersError, ServiceUnavailable, BrokenMPU, \
InvalidPartNumber, InvalidPartArgument
from swift.common.middleware.s3api.exception import NotS3Request
from swift.common.middleware.s3api.utils import utf8encode, \
S3Timestamp, mktime, MULTIUPLOAD_SUFFIX
@ -558,6 +559,57 @@ class S3Request(swob.Request):
# by full URL when absolute path given. See swift.swob for more detail.
self.environ['swift.leave_relative_location'] = True
def validate_part_number(self, parts_count=None, check_max=True):
"""
Get the partNumber param, if it exists, and check it is valid.
To be valid, a partNumber must satisfy two criteria. First, it must be
an integer between 1 and the maximum allowed parts, inclusive. The
maximum allowed parts is the maximum of the configured
``max_upload_part_num`` and, if given, ``parts_count``. Second, the
partNumber must be less than or equal to the ``parts_count``, if it is
given.
:param parts_count: if given, this is the number of parts in an
existing object.
:raises InvalidPartArgument: if the partNumber param is invalid i.e.
less than 1 or greater than the maximum allowed parts.
:raises InvalidPartNumber: if the partNumber param is valid but greater
than ``num_parts``.
:return: an integer part number if the partNumber param exists,
otherwise ``None``.
"""
part_number = self.params.get('partNumber')
if part_number is None:
return None
if self.range:
raise InvalidRequest('Cannot specify both Range header and '
'partNumber query parameter')
try:
parts_count = int(parts_count)
except (TypeError, ValueError):
# an invalid/empty param is treated like parts_count=max_parts
parts_count = self.conf.max_upload_part_num
# max_parts may be raised to the number of existing parts
max_parts = max(self.conf.max_upload_part_num, parts_count)
try:
part_number = int(part_number)
if part_number < 1:
raise ValueError
except ValueError:
raise InvalidPartArgument(max_parts, part_number) # 400
if check_max:
if part_number > max_parts:
raise InvalidPartArgument(max_parts, part_number) # 400
if part_number > parts_count:
raise InvalidPartNumber() # 416
return part_number
def check_signature(self, secret):
secret = utf8encode(secret)
user_signature = self.signature
@ -1044,7 +1096,10 @@ class S3Request(swob.Request):
if 'logging' in self.params:
return LoggingStatusController
if 'partNumber' in self.params:
if self.method == 'PUT':
return PartController
else:
return ObjectController
if 'uploadId' in self.params:
return UploadController
if 'uploads' in self.params:
@ -1315,7 +1370,6 @@ class S3Request(swob.Request):
'GET': {
HTTP_NOT_FOUND: not_found_handler,
HTTP_PRECONDITION_FAILED: PreconditionFailed,
HTTP_REQUESTED_RANGE_NOT_SATISFIABLE: InvalidRange,
},
'PUT': {
HTTP_NOT_FOUND: (NoSuchBucket, container),
@ -1414,7 +1468,7 @@ class S3Request(swob.Request):
raise InvalidArgument('X-Delete-At',
self.headers['X-Delete-At'],
err_str)
if 'X-Delete-After' in err_msg.decode('utf8'):
if 'X-Delete-After' in err_str:
raise InvalidArgument('X-Delete-After',
self.headers['X-Delete-After'],
err_str)
@ -1425,6 +1479,10 @@ class S3Request(swob.Request):
**self.signature_does_not_match_kwargs())
if status == HTTP_FORBIDDEN:
raise AccessDenied(reason='forbidden')
if status == HTTP_REQUESTED_RANGE_NOT_SATISFIABLE:
self.validate_part_number(
parts_count=resp.headers.get('x-amz-mp-parts-count'))
raise InvalidRange()
if status == HTTP_SERVICE_UNAVAILABLE:
raise ServiceUnavailable()
if status in (HTTP_RATE_LIMITED, HTTP_TOO_MANY_REQUESTS):

18
swift/common/middleware/s3api/s3response.py
View File

@ -72,6 +72,8 @@ def translate_swift_to_s3(key, val):
return key, val
elif _key == 'x-object-version-id':
return 'x-amz-version-id', val
elif _key == 'x-parts-count':
return 'x-amz-mp-parts-count', val
elif _key == 'x-copied-from-version-id':
return 'x-amz-copy-source-version-id', val
elif _key == 'x-backend-content-type' and \
@ -449,6 +451,17 @@ class InvalidObjectState(ErrorResponse):
_msg = 'The operation is not valid for the current state of the object.'
class InvalidPartArgument(InvalidArgument):
_code = 'InvalidArgument'
def __init__(self, max_parts, value):
err_msg = ('Part number must be an integer between '
'1 and %s, inclusive' % max_parts)
super(InvalidArgument, self).__init__(err_msg,
argument_name='partNumber',
argument_value=value)
class InvalidPart(ErrorResponse):
_status = '400 Bad Request'
_msg = 'One or more of the specified parts could not be found. The part ' \
@ -478,6 +491,11 @@ class InvalidRange(ErrorResponse):
_msg = 'The requested range cannot be satisfied.'
class InvalidPartNumber(ErrorResponse):
_status = '416 Requested Range Not Satisfiable'
_msg = 'The requested partnumber is not satisfiable'
class InvalidRequest(ErrorResponse):
_status = '400 Bad Request'
_msg = 'Invalid Request.'

1
swift/common/middleware/s3api/utils.py
View File

@ -172,6 +172,7 @@ class Config(dict):
'allow_no_owner': False,
'allowable_clock_skew': 900,
'ratelimit_as_client_error': False,
'max_upload_part_num': 1000,
}
def __init__(self, base=None):

20
test/s3api/__init__.py
View File

@ -151,12 +151,10 @@ class BaseS3TestCase(unittest.TestCase):
# Default to v4 signatures (as aws-cli does), but subclasses can override
signature_version = 's3v4'
@classmethod
def get_s3_client(cls, user):
return get_s3_client(user, cls.signature_version)
def get_s3_client(self, user):
return get_s3_client(user, self.signature_version)
@classmethod
def _remove_all_object_versions_from_bucket(cls, client, bucket_name):
def _remove_all_object_versions_from_bucket(self, client, bucket_name):
resp = client.list_object_versions(Bucket=bucket_name)
objs_to_delete = (resp.get('Versions', []) +
resp.get('DeleteMarkers', []))
@ -182,11 +180,10 @@ class BaseS3TestCase(unittest.TestCase):
objs_to_delete = (resp.get('Versions', []) +
resp.get('DeleteMarkers', []))
@classmethod
def clear_bucket(cls, client, bucket_name):
def clear_bucket(self, client, bucket_name):
timeout = time.time() + 10
backoff = 0.1
cls._remove_all_object_versions_from_bucket(client, bucket_name)
self._remove_all_object_versions_from_bucket(client, bucket_name)
try:
client.delete_bucket(Bucket=bucket_name)
except ClientError as e:
@ -199,7 +196,7 @@ class BaseS3TestCase(unittest.TestCase):
Bucket=bucket_name,
VersioningConfiguration={'Status': 'Suspended'})
while True:
cls._remove_all_object_versions_from_bucket(
self._remove_all_object_versions_from_bucket(
client, bucket_name)
# also try some version-unaware operations...
for key in client.list_objects(Bucket=bucket_name).get(
@ -224,13 +221,12 @@ class BaseS3TestCase(unittest.TestCase):
def create_name(self, slug):
return '%s%s-%s' % (TEST_PREFIX, slug, uuid.uuid4().hex)
@classmethod
def clear_account(cls, client):
def clear_account(self, client):
for bucket in client.list_buckets()['Buckets']:
if not bucket['Name'].startswith(TEST_PREFIX):
# these tests run against real s3 accounts
continue
cls.clear_bucket(client, bucket['Name'])
self.clear_bucket(client, bucket['Name'])
def tearDown(self):
client = self.get_s3_client(1)

481
test/s3api/test_mpu.py
View File

@ -17,7 +17,7 @@ from test.s3api import BaseS3TestCase
from botocore.exceptions import ClientError
class TestMultiPartUploads(BaseS3TestCase):
class BaseMultiPartUploadTestCase(BaseS3TestCase):
maxDiff = None
@ -26,10 +26,157 @@ class TestMultiPartUploads(BaseS3TestCase):
self.bucket_name = self.create_name('test-mpu')
resp = self.client.create_bucket(Bucket=self.bucket_name)
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
self.num_parts = 3
self.part_size = 5 * (2 ** 20) # 5 MB
def tearDown(self):
self.clear_bucket(self.client, self.bucket_name)
super(TestMultiPartUploads, self).tearDown()
super(BaseMultiPartUploadTestCase, self).tearDown()
def _make_part_bodies(self):
return [
('%d' % i) * self.part_size
for i in range(self.num_parts)
]
def _iter_part_num_ranges(self):
for i in range(self.num_parts):
start = self.part_size * i
end = start + self.part_size
# part_num is 1 indexed
yield i + 1, start, end
def _upload_mpu(self, key_name):
create_mpu_resp = self.client.create_multipart_upload(
Bucket=self.bucket_name, Key=key_name)
self.assertEqual(200, create_mpu_resp[
'ResponseMetadata']['HTTPStatusCode'])
upload_id = create_mpu_resp['UploadId']
part_bodies = self._make_part_bodies()
parts = []
for i, body in enumerate(part_bodies, 1):
part_resp = self.client.upload_part(
Body=body, Bucket=self.bucket_name, Key=key_name,
PartNumber=i, UploadId=upload_id)
self.assertEqual(200, part_resp[
'ResponseMetadata']['HTTPStatusCode'])
parts.append({
'ETag': part_resp['ETag'],
'PartNumber': i,
})
# this helper doesn't bother calling list-parts, it's not required
# and we know what we uploaded
complete_mpu_resp = self.client.complete_multipart_upload(
Bucket=self.bucket_name, Key=key_name,
MultipartUpload={
'Parts': parts,
},
UploadId=upload_id,
)
self.assertEqual(200, complete_mpu_resp[
'ResponseMetadata']['HTTPStatusCode'])
return complete_mpu_resp
def upload_mpu_version(self, key_name):
complete_mpu_resp = self._upload_mpu(key_name)
# AWS returns the version_id *in* the MPU-complete response but s3api
# does NOT (see https://bugs.launchpad.net/swift/+bug/2043619), so we
# do an extra HEAD to get the version
head_object_resp = self.client.head_object(
Bucket=self.bucket_name, Key=key_name)
self.assertEqual(200, head_object_resp[
'ResponseMetadata']['HTTPStatusCode'])
return complete_mpu_resp['ETag'], head_object_resp.get('VersionId')
def upload_mpu(self, key_name):
complete_mpu_resp = self._upload_mpu(key_name)
return complete_mpu_resp['ETag']
def _verify_part_num_response(self, method, key_name, mpu_etag,
version=None):
part_bodies = self._make_part_bodies()
total_size = self.num_parts * self.part_size
for part_num, start, end in self._iter_part_num_ranges():
extra_kwargs = {}
if version is not None:
extra_kwargs['VersionId'] = version
resp = method(Bucket=self.bucket_name, Key=key_name,
PartNumber=part_num, **extra_kwargs)
self.assertEqual(206, resp['ResponseMetadata'][
'HTTPStatusCode'])
self.assertEqual(self.part_size, resp['ContentLength'])
if method == self.client.get_object:
resp_body = b''.join(resp['Body']).decode()
# our part_bodies are zero indexed
self.assertEqual(resp_body, part_bodies[part_num - 1])
expected_range = 'bytes %s-%s/%s' % (
start, end - 1, total_size)
self.assertEqual(expected_range, resp['ContentRange'])
# ETag and PartsCount are from the MPU
self.assertEqual(mpu_etag, resp['ETag'], mpu_etag)
self.assertEqual(self.num_parts, resp['PartsCount'])
self.assertEqual('bytes', resp['AcceptRanges'])
if version is None:
self.assertNotIn('VersionId', resp)
else:
self.assertEqual(version, resp['VersionId'])
def _verify_copy_parts(self, key_src, key_dest, upload_id):
parts = []
for part_num, start, end in self._iter_part_num_ranges():
copy_range = 'bytes=%d-%d' % (start, end - 1)
copy_resp = self.client.\
upload_part_copy(Bucket=self.bucket_name,
Key=key_dest, PartNumber=part_num,
CopySource={
'Bucket': self.bucket_name,
'Key': key_src,
}, CopySourceRange=copy_range,
UploadId=upload_id)
self.assertEqual(200, copy_resp[
'ResponseMetadata']['HTTPStatusCode'])
self.assertTrue(copy_resp['CopyPartResult']['ETag'])
self.assertTrue(copy_resp['CopyPartResult']['LastModified'])
parts.append({
'ETag': copy_resp['CopyPartResult']['ETag'],
'PartNumber': part_num,
})
complete_mpu_resp = self.client.complete_multipart_upload(
Bucket=self.bucket_name, Key=key_dest,
MultipartUpload={
'Parts': parts,
},
UploadId=upload_id,
)
self.assertEqual(200, complete_mpu_resp[
'ResponseMetadata']['HTTPStatusCode'])
return complete_mpu_resp['ETag']
class TestMultiPartUpload(BaseMultiPartUploadTestCase):
def setUp(self):
super(TestMultiPartUpload, self).setUp()
def _discover_max_part_num(self):
key_name = self.create_name('discover-max-part-num')
self.upload_mpu(key_name)
with self.assertRaises(ClientError) as cm:
self.client.get_object(Bucket=self.bucket_name,
Key=key_name, PartNumber=0)
err_resp = cm.exception.response
self.assertEqual(400, err_resp['ResponseMetadata']['HTTPStatusCode'])
self.assertEqual('InvalidArgument', err_resp['Error']['Code'])
err_msg = err_resp['Error']['Message']
preamble = 'Part number must be an integer between 1 and '
self.assertIn(preamble, err_msg)
return int(err_msg[len(preamble):].split(',')[0])
def test_basic_upload(self):
key_name = self.create_name('key')
@ -68,6 +215,303 @@ class TestMultiPartUploads(BaseS3TestCase):
self.assertEqual(200, complete_mpu_resp[
'ResponseMetadata']['HTTPStatusCode'])
def _check_part_num_invalid_exc(self, exc, val, max_part_num,
is_head=False):
err_resp = exc.response
self.assertEqual(400, err_resp['ResponseMetadata']['HTTPStatusCode'])
if is_head:
err_code = '400'
err_msg = 'Bad Request'
else:
err_code = 'InvalidArgument'
err_msg = 'Part number must be an integer between ' \
'1 and %d, inclusive' % max_part_num
self.assertEqual(err_code, err_resp['Error']['Code'], err_resp)
self.assertEqual(err_msg, err_resp['Error']['Message'])
if is_head:
self.assertNotIn('ArgumentName', err_resp['Error'])
self.assertNotIn('ArgumentValue', err_resp['Error'])
else:
self.assertEqual('partNumber', err_resp['Error']['ArgumentName'])
self.assertEqual(str(val), err_resp['Error']['ArgumentValue'])
def _check_part_num_out_of_range_exc(self, exc, is_head=False):
err_resp = exc.response
self.assertEqual(416, err_resp['ResponseMetadata']['HTTPStatusCode'])
if is_head:
err_code = '416'
err_msg = 'Requested Range Not Satisfiable'
else:
err_code = 'InvalidPartNumber'
err_msg = 'The requested partnumber is not satisfiable'
self.assertEqual(err_code, err_resp['Error']['Code'], err_resp)
self.assertEqual(err_msg, err_resp['Error']['Message'], err_resp)
def test_get_object_partNumber_errors(self):
max_part_num = self._discover_max_part_num()
key_name = self.create_name('invalid-part-num-test')
mpu_etag = self.upload_mpu(key_name)
# partNumber argument is 1 indexed
with self.assertRaises(ClientError) as caught:
self.client.get_object(Bucket=self.bucket_name,
Key=key_name, PartNumber=0)
self._check_part_num_invalid_exc(caught.exception, 0, max_part_num)
# all other partNumber args are valid
self._verify_part_num_response(
self.client.get_object, key_name, mpu_etag)
with self.assertRaises(ClientError) as caught:
self.client.get_object(Bucket=self.bucket_name,
Key=key_name, PartNumber=self.num_parts + 1)
self._check_part_num_out_of_range_exc(caught.exception)
with self.assertRaises(ClientError) as caught:
self.client.get_object(Bucket=self.bucket_name,
Key=key_name, PartNumber=max_part_num)
self._check_part_num_out_of_range_exc(caught.exception)
# because of ParamValidationError we can't test 'foo'
val = -1
with self.assertRaises(ClientError) as caught:
self.client.get_object(Bucket=self.bucket_name,
Key=key_name, PartNumber=val)
self._check_part_num_invalid_exc(caught.exception, val, max_part_num)
val = max_part_num + 1
with self.assertRaises(ClientError) as caught:
self.client.get_object(Bucket=self.bucket_name,
Key=key_name, PartNumber=val)
self._check_part_num_invalid_exc(caught.exception, val, max_part_num)
def test_head_object_partNumber_errors(self):
max_part_num = self._discover_max_part_num()
key_name = self.create_name('invalid-part-num-head')
mpu_etag = self.upload_mpu(key_name)
# partNumber argument is 1 indexed
with self.assertRaises(ClientError) as caught:
self.client.head_object(Bucket=self.bucket_name,
Key=key_name, PartNumber=0)
self._check_part_num_invalid_exc(caught.exception, 0, max_part_num,
is_head=True)
# all other partNumber args are valid
self._verify_part_num_response(
self.client.head_object, key_name, mpu_etag)
with self.assertRaises(ClientError) as caught:
self.client.head_object(Bucket=self.bucket_name, Key=key_name,
PartNumber=self.num_parts + 1)
self._check_part_num_out_of_range_exc(caught.exception, is_head=True)
with self.assertRaises(ClientError) as caught:
self.client.head_object(Bucket=self.bucket_name, Key=key_name,
PartNumber=max_part_num)
self._check_part_num_out_of_range_exc(caught.exception, is_head=True)
# because of ParamValidationError we can't test 'foo'
val = -1
with self.assertRaises(ClientError) as caught:
self.client.head_object(Bucket=self.bucket_name, Key=key_name,
PartNumber=val)
self._check_part_num_invalid_exc(caught.exception, val, max_part_num,
is_head=True)
val = max_part_num + 1
with self.assertRaises(ClientError) as caught:
self.client.head_object(Bucket=self.bucket_name, Key=key_name,
PartNumber=val)
self._check_part_num_invalid_exc(caught.exception, val, max_part_num,
is_head=True)
def test_part_number_non_mpu(self):
max_part_num = self._discover_max_part_num()
key_name = self.create_name('part-num-non-mpu')
self.client.put_object(Bucket=self.bucket_name,
Key=key_name,
Body=b'non-mpu-object')
head_resp = self.client.head_object(Bucket=self.bucket_name,
Key=key_name)
# sanity check
self.assertEqual(200,
head_resp['ResponseMetadata']['HTTPStatusCode'])
self.assertEqual(head_resp['AcceptRanges'], 'bytes')
self.assertEqual(head_resp['ContentLength'], 14)
head_resp = self.client.head_object(Bucket=self.bucket_name,
Key=key_name,
PartNumber=1)
self.assertEqual(206,
head_resp['ResponseMetadata']['HTTPStatusCode'])
self.assertEqual(head_resp['ContentLength'], 14)
get_resp = self.client.get_object(Bucket=self.bucket_name,
Key=key_name,
PartNumber=1)
self.assertEqual(206,
get_resp['ResponseMetadata']['HTTPStatusCode'])
self.assertEqual(get_resp['ContentLength'], 14)
self.assertEqual(get_resp['ContentRange'], 'bytes 0-13/14')
self.assertEqual(b'non-mpu-object', b''.join(get_resp['Body']))
with self.assertRaises(ClientError) as caught:
self.client.get_object(Bucket=self.bucket_name,
Key=key_name,
PartNumber=4)
self._check_part_num_out_of_range_exc(caught.exception)
with self.assertRaises(ClientError) as caught:
self.client.head_object(Bucket=self.bucket_name,
Key=key_name,
PartNumber=4)
self._check_part_num_out_of_range_exc(caught.exception, is_head=True)
with self.assertRaises(ClientError) as caught:
self.client.get_object(Bucket=self.bucket_name,
Key=key_name,
PartNumber=0)
self._check_part_num_invalid_exc(caught.exception, 0, max_part_num)
with self.assertRaises(ClientError) as caught:
self.client.head_object(Bucket=self.bucket_name,
Key=key_name,
PartNumber=0)
self._check_part_num_invalid_exc(caught.exception, 0, max_part_num,
is_head=True)
invalid_part_num = 10001
with self.assertRaises(ClientError) as caught:
self.client.get_object(Bucket=self.bucket_name,
Key=key_name,
PartNumber=invalid_part_num)
self._check_part_num_invalid_exc(caught.exception, invalid_part_num,
max_part_num)
with self.assertRaises(ClientError) as caught:
self.client.head_object(Bucket=self.bucket_name,
Key=key_name,
PartNumber=invalid_part_num)
self._check_part_num_invalid_exc(caught.exception, invalid_part_num,
max_part_num, is_head=True)
def test_get_object_partNumber_and_range(self):
# partNumber not allowed with Range even for non-mpu object
key_name = self.create_name('part-num-mpu')
self._upload_mpu(key_name)
with self.assertRaises(ClientError) as caught:
self.client.get_object(Bucket=self.bucket_name,
Key=key_name,
PartNumber=1,
Range='bytes=1-2')
err_resp = caught.exception.response
self.assertEqual(400, err_resp['ResponseMetadata']['HTTPStatusCode'])
self.assertEqual('InvalidRequest', err_resp['Error']['Code'], err_resp)
self.assertEqual('Cannot specify both Range header and partNumber '
'query parameter', err_resp['Error']['Message'])
key_name = self.create_name('part-num-non-mpu')
self.client.put_object(Bucket=self.bucket_name,
Key=key_name,
Body=b'non-mpu-object')
with self.assertRaises(ClientError) as caught:
self.client.get_object(Bucket=self.bucket_name,
Key=key_name,
PartNumber=1,
Range='bytes=1-2')
err_resp = caught.exception.response
self.assertEqual(400, err_resp['ResponseMetadata']['HTTPStatusCode'])
self.assertEqual('InvalidRequest', err_resp['Error']['Code'], err_resp)
self.assertEqual('Cannot specify both Range header and partNumber '
'query parameter', err_resp['Error']['Message'])
# partNumber + Range error trumps bad partNumber
with self.assertRaises(ClientError) as caught:
self.client.get_object(Bucket=self.bucket_name,
Key=key_name,
PartNumber=0,
Range='bytes=1-2')
err_resp = caught.exception.response
self.assertEqual(400, err_resp['ResponseMetadata']['HTTPStatusCode'])
self.assertEqual('InvalidRequest', err_resp['Error']['Code'], err_resp)
self.assertEqual('Cannot specify both Range header and partNumber '
'query parameter', err_resp['Error']['Message'])
def test_upload_part_copy(self):
self.num_parts = 4
key_src = self.create_name('part-copy-src')
key_dest = self.create_name('part-copy-dest')
mpu_etag_src = self.upload_mpu(key_src)
self._verify_part_num_response(
self.client.get_object, key_src, mpu_etag_src)
self._verify_part_num_response(
self.client.head_object, key_src, mpu_etag_src)
create_mpu_dest = self.client.create_multipart_upload(
Bucket=self.bucket_name, Key=key_dest)
self.assertEqual(200, create_mpu_dest[
'ResponseMetadata']['HTTPStatusCode'])
upload_id = create_mpu_dest['UploadId']
mpu_etag_dst = self._verify_copy_parts(key_src, key_dest, upload_id)
self._verify_part_num_response(
self.client.get_object, key_dest, mpu_etag_dst)
self._verify_part_num_response(
self.client.head_object, key_dest, mpu_etag_dst)
def test_copy_mpu_from_parts(self):
key_src = self.create_name('copy-from-from-src')
mpu_etag_src = self.upload_mpu(key_src)
# client wanting to copy object would first HEAD
head_object_resp = self.client.head_object(
Bucket=self.bucket_name, Key=key_src)
# the client will know it's an mpu and how many parts
self.assertEqual(mpu_etag_src, head_object_resp['ETag'])
self.assertIn('-', mpu_etag_src)
num_parts = int(mpu_etag_src.strip('"').rsplit('-')[-1])
# create new mpu
key_dest = self.create_name('copy-from-from-dest')
create_mpu_dest = self.client.create_multipart_upload(
Bucket=self.bucket_name, Key=key_dest)
self.assertEqual(200, create_mpu_dest[
'ResponseMetadata']['HTTPStatusCode'])
upload_id = create_mpu_dest['UploadId']
parts = []
start = 0
# do HEAD?partNumber to get copy range
for part_num in range(1, num_parts + 1):
part_head_resp = self.client.head_object(
Bucket=self.bucket_name, Key=key_src, PartNumber=part_num)
end = start + part_head_resp['ContentLength']
copy_range = 'bytes=%s-%s' % (start, end - 1)
copy_resp = self.client.upload_part_copy(
Bucket=self.bucket_name, Key=key_dest, PartNumber=part_num,
CopySource={
'Bucket': self.bucket_name,
'Key': key_src,
},
CopySourceRange=copy_range, UploadId=upload_id)
self.assertEqual(200, copy_resp[
'ResponseMetadata']['HTTPStatusCode'])
parts.append({
'ETag': copy_resp['CopyPartResult']['ETag'],
'PartNumber': part_num,
})
start = end
complete_mpu_resp = self.client.complete_multipart_upload(
Bucket=self.bucket_name, Key=key_dest,
MultipartUpload={
'Parts': parts,
},
UploadId=upload_id,
)
self.assertEqual(200, complete_mpu_resp[
'ResponseMetadata']['HTTPStatusCode'])
self.assertEqual(complete_mpu_resp['ETag'], mpu_etag_src)
def test_create_list_abort_multipart_uploads(self):
key_name = self.create_name('key')
create_mpu_resp = self.client.create_multipart_upload(
@ -138,3 +582,36 @@ class TestMultiPartUploads(BaseS3TestCase):
self.assertEqual(complete_mpu_resp['Error']['UploadId'], upload_id)
self.assertIn(complete_mpu_resp['Error']['PartNumber'], ('1', '2'))
self.assertEqual(complete_mpu_resp['Error']['ETag'], None)
class TestVersionedMultiPartUpload(BaseMultiPartUploadTestCase):
def setUp(self):
super(TestVersionedMultiPartUpload, self).setUp()
resp = self.client.put_bucket_versioning(
Bucket=self.bucket_name,
VersioningConfiguration={'Status': 'Enabled'})
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
def tearDown(self):
resp = self.client.put_bucket_versioning(
Bucket=self.bucket_name,
VersioningConfiguration={'Status': 'Suspended'})
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
super(TestVersionedMultiPartUpload, self).tearDown()
def test_get_by_part_number_with_versioning(self):
# create 3 version with progressively larger sizes
parts_counts = [2, 3, 4]
key_name = self.create_name('part-num-versions')
version_vars = []
for num_parts in parts_counts:
self.num_parts = num_parts
etag, version_id = self.upload_mpu_version(key_name)
version_vars.append((num_parts, etag, version_id))
for num_parts, mpu_etag, version in version_vars:
self.num_parts = num_parts
self._verify_part_num_response(
self.client.get_object, key_name, mpu_etag, version)
self._verify_part_num_response(
self.client.head_object, key_name, mpu_etag, version)

27
test/unit/common/middleware/s3api/__init__.py
View File

@ -28,19 +28,16 @@ from swift.common.middleware.s3api.etree import fromstring
from swift.common.middleware.s3api.subresource import Owner, encode_acl, \
Grant, User, ACL, PERMISSIONS, AllUsers, AuthenticatedUsers
from test.debug_logger import debug_logger
from test.unit.common.middleware.helpers import FakeSwift
class FakeApp(object):
class FakeAuthApp(object):
container_existence_skip_cache = 0.0
account_existence_skip_cache = 0.0
def __init__(self):
def __init__(self, app):
self.remote_user = 'authorized'
self._pipeline_final_app = self
self.swift = FakeSwift()
self.logger = debug_logger()
self.app = app
def _update_s3_path_info(self, env):
"""
@ -82,7 +79,7 @@ class FakeApp(object):
def __call__(self, env, start_response):
self.handle(env)
return self.swift(env, start_response)
return self.app(env, start_response)
class S3ApiTestCase(unittest.TestCase):
@ -90,6 +87,9 @@ class S3ApiTestCase(unittest.TestCase):
def __init__(self, name):
unittest.TestCase.__init__(self, name)
def _wrap_app(self, app):
return FakeAuthApp(app)
def setUp(self):
# setup default config dict
self.conf = {
@ -110,12 +110,13 @@ class S3ApiTestCase(unittest.TestCase):
'log_level': 'debug'
}
self.app = FakeApp()
self.swift = self.app.swift
# note: self.conf has no __file__ key so check_pipeline will be skipped
# when constructing self.s3api
self.swift = FakeSwift()
self.app = self._wrap_app(self.swift)
self.app._pipeline_final_app = self.swift
self.s3api = filter_factory({}, **self.conf)(self.app)
self.logger = self.s3api.logger = self.swift.logger = debug_logger()
self.logger = self.s3api.logger = self.swift.logger
# if you change the registered acl response for /bucket or
# /bucket/object tearDown will complain at you; you can set this to
@ -282,11 +283,11 @@ class S3ApiTestCaseAcl(S3ApiTestCase):
self.swift.register('GET', path, swob.HTTPOk, {}, json.dumps([])),
# setup sticky ACL headers...
grants = [_gen_grant(perm) for perm in PERMISSIONS]
self.grants = [_gen_grant(perm) for perm in PERMISSIONS]
self.default_owner = Owner('test:tester', 'test:tester')
container_headers = _gen_test_headers(self.default_owner, grants)
container_headers = _gen_test_headers(self.default_owner, self.grants)
object_headers = _gen_test_headers(
self.default_owner, grants, 'object')
self.default_owner, self.grants, 'object')
public_headers = _gen_test_headers(
self.default_owner, [Grant(AllUsers(), 'READ')])
authenticated_headers = _gen_test_headers(

661
test/unit/common/middleware/s3api/test_multi_get.py Normal file
View File

@ -0,0 +1,661 @@
# Copyright (c) 2023 NVIDIA
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import binascii
import string
import json
import mock
from swift.common import swob, utils
from swift.common.request_helpers import get_reserved_name
from swift.common.middleware import symlink
from swift.common.middleware.versioned_writes import object_versioning as ov
from test.unit import make_timestamp_iter
from test.unit.common.middleware.test_slo import slo, md5hex
from test.unit.common.middleware.s3api import (
S3ApiTestCase, S3ApiTestCaseAcl, _gen_test_headers)
def _prepare_mpu(swift, ts_iter, upload_id, num_segments,
segment_bucket='bucket+segments', segment_key='mpu'):
manifest = []
for i, letter in enumerate(string.ascii_lowercase):
if len(manifest) >= num_segments:
break
size = (i + 1) * 5
body = letter * size
etag = md5hex(body)
path = '/%s/%s/%s/%s' % (segment_bucket, segment_key, upload_id, i + 1)
swift.register('GET', '/v1/AUTH_test' + path, swob.HTTPOk, {
'Content-Length': len(body),
'Etag': etag,
}, body)
manifest.append({
"name": path,
"bytes": size,
"hash": etag,
"content_type": "application/octet-stream",
"last_modified": next(ts_iter).isoformat,
})
slo_etag = md5hex(''.join(s['hash'] for s in manifest))
s3_hash = md5hex(binascii.a2b_hex(''.join(
s['hash'] for s in manifest)))
s3_etag = "%s-%s" % (s3_hash, len(manifest))
manifest_json = json.dumps(manifest)
json_md5 = md5hex(manifest_json)
manifest_headers = {
'Content-Length': str(len(manifest_json)),
'X-Static-Large-Object': 'true',
'Etag': json_md5,
'Content-Type': 'application/octet-stream',
'X-Object-Sysmeta-Slo-Etag': slo_etag,
'X-Object-Sysmeta-Slo-Size': str(sum(
s['bytes'] for s in manifest)),
'X-Object-Sysmeta-S3Api-Etag': s3_etag,
'X-Object-Sysmeta-S3Api-Upload-Id': upload_id,
'X-Object-Sysmeta-Container-Update-Override-Etag':
'%s; s3_etag=%s; slo_etag=%s' % (json_md5, s3_etag, slo_etag),
}
return manifest_headers, manifest_json
class TestMpuGETorHEAD(S3ApiTestCase):
def _wrap_app(self, app):
self.slo = slo.filter_factory({'rate_limit_under_size': '0'})(app)
return super(TestMpuGETorHEAD, self)._wrap_app(self.slo)
def setUp(self):
# this will call our _wrap_app
super(TestMpuGETorHEAD, self).setUp()
self.ts = make_timestamp_iter()
manifest_headers, manifest_json = _prepare_mpu(
self.swift, self.ts, 'X', 3)
self.s3_etag = manifest_headers['X-Object-Sysmeta-S3Api-Etag']
self.swift.register(
'GET', '/v1/AUTH_test/bucket/mpu',
swob.HTTPOk, manifest_headers, manifest_json.encode('ascii'))
self.s3_acl = False
def test_mpu_GET(self):
req = swob.Request.blank('/bucket/mpu', headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()
})
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
self.assertEqual(body, b'aaaaabbbbbbbbbbccccccccccccccc')
expected_calls = [
('GET', '/v1/AUTH_test/bucket/mpu'),
('GET', '/v1/AUTH_test/bucket+segments/mpu/X/1'
'?multipart-manifest=get'),
('GET', '/v1/AUTH_test/bucket+segments/mpu/X/2'
'?multipart-manifest=get'),
('GET', '/v1/AUTH_test/bucket+segments/mpu/X/3'
'?multipart-manifest=get'),
]
if self.s3_acl:
# pre-flight object ACL check
expected_calls.insert(0, ('HEAD', '/v1/AUTH_test/bucket/mpu'))
self.assertEqual(self.swift.calls, expected_calls)
self.assertEqual(headers['Content-Length'], '30')
self.assertEqual(headers['Etag'], '"%s"' % self.s3_etag)
self.assertNotIn('X-Amz-Mp-Parts-Count', headers)
def test_mpu_GET_part_num(self):
req = swob.Request.blank('/bucket/mpu', params={
'partNumber': '2',
}, headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()
})
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '206')
self.assertEqual(body, b'bbbbbbbbbb')
expected_calls = [
('GET', '/v1/AUTH_test/bucket/mpu?part-number=2'),
('GET', '/v1/AUTH_test/bucket+segments/mpu/X/2'
'?multipart-manifest=get'),
]
if self.s3_acl:
expected_calls.insert(0, ('HEAD', '/v1/AUTH_test/bucket/mpu'))
self.assertEqual(self.swift.calls, expected_calls)
self.assertEqual(headers['Content-Length'], '10')
self.assertEqual(headers['Content-Range'], 'bytes 5-14/30')
self.assertEqual(headers['Etag'], '"%s"' % self.s3_etag)
self.assertEqual(headers['X-Amz-Mp-Parts-Count'], '3')
def test_mpu_GET_invalid_part_num(self):
req = swob.Request.blank('/bucket/mpu', params={
'partNumber': 'foo',
}, headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()
})
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '400')
self.assertEqual(self._get_error_code(body), 'InvalidArgument')
self.assertEqual(self.swift.calls, [])
def test_mpu_GET_zero_part_num(self):
req = swob.Request.blank('/bucket/mpu', params={
'partNumber': '0',
}, headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()
})
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '400')
self.assertEqual(self._get_error_code(body), 'InvalidArgument')
self.assertEqual(self.swift.calls, [])
def _do_test_mpu_GET_out_of_range_part_num(self, part_number):
self.swift.clear_calls()
req = swob.Request.blank('/bucket/mpu', params={
'partNumber': str(part_number),
}, headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()
})
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '416')
self.assertEqual(self._get_error_code(body), 'InvalidPartNumber')
expected_calls = [
# s3api.controller.obj doesn't know yet if it's SLO, we delegate
# param validation
('GET', '/v1/AUTH_test/bucket/mpu?part-number=%s' % part_number),
]
if self.s3_acl:
expected_calls.insert(0, ('HEAD', '/v1/AUTH_test/bucket/mpu'))
self.assertEqual(self.swift.calls, expected_calls)
def test_mpu_GET_out_of_range_part_num(self):
self._do_test_mpu_GET_out_of_range_part_num(4)
self._do_test_mpu_GET_out_of_range_part_num(10000)
def test_existing_part_number_greater_than_max_parts_allowed(self):
part_number = 3
max_parts = 2
req = swob.Request.blank('/bucket/mpu', params={
'partNumber': str(part_number),
}, headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()
})
bad_req = swob.Request.blank('/bucket/mpu', params={
'partNumber': str(part_number + 1),
}, headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()
})
with mock.patch.object(self.s3api.conf,
'max_upload_part_num', max_parts):
# num_parts >= part number > max parts
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '206')
# part number > num parts > max parts
status, headers, body = self.call_s3api(bad_req)
self.assertEqual(status.split()[0], '400')
self.assertIn('must be an integer between 1 and 3, inclusive',
self._get_error_message(body))
max_parts = part_number + 1
with mock.patch.object(self.s3api.conf,
'max_upload_part_num', max_parts):
# max_parts > num_parts >= part number
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '206')
# max_parts >= part number > num parts
status, headers, body = self.call_s3api(bad_req)
self.assertEqual(status.split()[0], '416')
self.assertIn('The requested partnumber is not satisfiable',
self._get_error_message(body))
# part number > max_parts > num parts
bad_req.params = {'partNumber': str(max_parts + 1)}
status, headers, body = self.call_s3api(bad_req)
self.assertEqual(status.split()[0], '400')
self.assertIn('must be an integer between 1 and 4, inclusive',
self._get_error_message(body))
def test_mpu_GET_huge_part_num(self):
req = swob.Request.blank('/bucket/mpu', params={
'partNumber': '10001',
}, headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()
})
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '400')
self.assertEqual(self._get_error_code(body), 'InvalidArgument')
expected_calls = [
# XXX is this value configurable? do we need the SLO request?
('GET', '/v1/AUTH_test/bucket/mpu?part-number=10001'),
]
if self.s3_acl:
expected_calls.insert(0, ('HEAD', '/v1/AUTH_test/bucket/mpu'))
self.assertEqual(self.swift.calls, expected_calls)
def test_mpu_HEAD_part_num(self):
req = swob.Request.blank('/bucket/mpu', params={
'partNumber': '1',
}, headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()
}, method='HEAD')
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '206')
self.assertEqual(body, b'')
self.assertEqual(self.swift.calls, [
('HEAD', '/v1/AUTH_test/bucket/mpu?part-number=1'),
('GET', '/v1/AUTH_test/bucket/mpu?part-number=1'),
])
self.assertEqual(headers['Content-Length'], '5')
self.assertEqual(headers['Content-Range'], 'bytes 0-4/30')
self.assertEqual(headers['Etag'], '"%s"' % self.s3_etag)
self.assertEqual(headers['X-Amz-Mp-Parts-Count'], '3')
def test_mpu_HEAD_invalid_part_num(self):
req = swob.Request.blank('/bucket/mpu', method='HEAD', params={
'partNumber': 'foo',
}, headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()
})
status, headers, _ = self.call_s3api(req)
self.assertEqual(status.split()[0], '400')
self.assertEqual(self.swift.calls, [])
def test_mpu_HEAD_zero_part_num(self):
req = swob.Request.blank('/bucket/mpu', method='HEAD', params={
'partNumber': '0',
}, headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()
})
status, headers, _ = self.call_s3api(req)
self.assertEqual(status.split()[0], '400')
self.assertEqual(self.swift.calls, [])
def _do_test_mpu_HEAD_out_of_range_part_num(self, part_number):
self.swift.clear_calls()
req = swob.Request.blank('/bucket/mpu', method='HEAD', params={
'partNumber': str(part_number),
}, headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()
})
status, headers, _ = self.call_s3api(req)
self.assertEqual(status.split()[0], '416')
self.assertEqual(self.swift.calls, [
('HEAD', '/v1/AUTH_test/bucket/mpu?part-number=%s' % part_number),
# SLO has to refetch to *see* if it's out-of-bounds
('GET', '/v1/AUTH_test/bucket/mpu?part-number=%s' % part_number),
])
def test_mpu_HEAD_out_of_range_part_num(self):
self._do_test_mpu_HEAD_out_of_range_part_num(4)
self._do_test_mpu_HEAD_out_of_range_part_num(10000)
def test_mpu_HEAD_huge_part_num(self):
req = swob.Request.blank('/bucket/mpu', method='HEAD', params={
'partNumber': '10001',
}, headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()
})
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '400')
self.assertEqual(self.swift.calls, [
('HEAD', '/v1/AUTH_test/bucket/mpu?part-number=10001'),
# XXX were two requests worth it to 400?
# how big can you configure SLO?
# do such manifests *exist*?
('GET', '/v1/AUTH_test/bucket/mpu?part-number=10001'),
])
class TestMpuGETorHEADAcl(TestMpuGETorHEAD, S3ApiTestCaseAcl):
def setUp(self):
super(TestMpuGETorHEADAcl, self).setUp()
object_headers = _gen_test_headers(
self.default_owner, self.grants, 'object')
self.swift.update_sticky_response_headers(
'/v1/AUTH_test/bucket/mpu', object_headers)
# this is used to flag insertion of expected HEAD pre-flight request of
# object ACLs
self.s3_acl = True
class TestVersionedMpuGETorHEAD(S3ApiTestCase):
def _wrap_app(self, app):
self.sym = symlink.filter_factory({})(app)
self.sym.logger = self.swift.logger
self.ov = ov.ObjectVersioningMiddleware(self.sym, {})
self.ov.logger = self.swift.logger
self.slo = slo.filter_factory({'rate_limit_under_size': '0'})(self.ov)
self.slo.logger = self.swift.logger
return super(TestVersionedMpuGETorHEAD, self)._wrap_app(self.slo)
def setUp(self):
# this will call our _wrap_app
super(TestVersionedMpuGETorHEAD, self).setUp()
self.ts = make_timestamp_iter()
self.swift.register('HEAD', '/v1/AUTH_test/bucket+segments',
swob.HTTPNoContent, {}, None)
versions_container = get_reserved_name('versions', 'bucket')
self.swift.register(
'HEAD', '/v1/AUTH_test/bucket', swob.HTTPNoContent, {
ov.SYSMETA_VERSIONS_CONT: versions_container,
ov.SYSMETA_VERSIONS_ENABLED: True,
}, None)
self.swift.register('HEAD', '/v1/AUTH_test/%s' % versions_container,
swob.HTTPNoContent, {}, None)
num_versions = 3
self.version_ids = []
for v in range(num_versions):
upload_id = 'X%s' % v
num_segments = 3 + v
manifest_headers, manifest_json = _prepare_mpu(
self.swift, self.ts, upload_id, num_segments)
version_ts = next(self.ts)
# add in a little user-meta to keep versions stright
manifest_version_headers = dict(manifest_headers, **{
'x-object-meta-user-notes': 'version%s' % v,
'x-backend-timestamp': version_ts.internal,
})
self.version_ids.append(version_ts.normal)
obj_version_path = get_reserved_name('mpu', (~version_ts).normal)
self.swift.register(
'GET', '/v1/AUTH_test/%s/%s' % (
versions_container, obj_version_path),
swob.HTTPOk, manifest_version_headers,
manifest_json.encode('ascii'))
# TODO: make a current version symlink
symlink_target = '%s/%s' % (versions_container, obj_version_path)
slo_etag = manifest_headers['X-Object-Sysmeta-Slo-Etag']
s3_etag = manifest_headers['X-Object-Sysmeta-S3Api-Etag']
symlink_target_etag = json_md5 = manifest_headers['Etag']
symlink_target_bytes = manifest_headers['X-Object-Sysmeta-Slo-Size']
manifest_symlink_headers = dict(manifest_headers, **{
'X-Object-Sysmeta-Container-Update-Override-Etag':
'%s; s3_etag=%s; slo_etag=%s; symlink_target=%s; '
'symlink_target_etag=%s; symlink_target_bytes=%s' % (
json_md5, s3_etag, slo_etag, symlink_target,
symlink_target_etag, symlink_target_bytes),
'X-Object-Sysmeta-Allow-Reserved-Names': 'true',
'X-Object-Sysmeta-Symlink-Target': symlink_target,
'X-Object-Sysmeta-Symlink-Target-Bytes': str(symlink_target_bytes),
'X-Object-Sysmeta-Symlink-Target-Etag': symlink_target_etag,
'X-Object-Sysmeta-Symloop-Extend': 'true',
'X-Object-Sysmeta-Versions-Symlink': 'true',
})
self.swift.register(
'GET', '/v1/AUTH_test/bucket/mpu', swob.HTTPOk,
manifest_symlink_headers, '')
self.s3_acl = False
def test_mpu_GET_version(self):
req = swob.Request.blank('/bucket/mpu', params={
'versionId': self.version_ids[0],
}, headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()
})
status, headers, body = self.call_s3api(req)
self.assertEqual(status, '200 OK')
self.assertEqual(headers['x-amz-meta-user-notes'], 'version0')
self.assertEqual(headers['Content-Length'], '30')
self.assertEqual(body, b'aaaaabbbbbbbbbbccccccccccccccc')
expected_calls = [
('HEAD', '/v1/AUTH_test'),
('HEAD', '/v1/AUTH_test/bucket'),
('HEAD', '/v1/AUTH_test/\x00versions\x00bucket'),
('GET', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s'
'?version-id=%s' % (
(~utils.Timestamp(self.version_ids[0])).normal,
self.version_ids[0])),
('HEAD', '/v1/AUTH_test/bucket+segments'),
('GET', '/v1/AUTH_test/bucket+segments/mpu/X0/1'
'?multipart-manifest=get'),
('GET', '/v1/AUTH_test/bucket+segments/mpu/X0/2'
'?multipart-manifest=get'),
('GET', '/v1/AUTH_test/bucket+segments/mpu/X0/3'
'?multipart-manifest=get')
]
if self.s3_acl:
expected_calls.insert(3, (
'HEAD', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s'
'?version-id=%s' % (
(~utils.Timestamp(self.version_ids[0])).normal,
self.version_ids[0])
))
self.assertEqual(self.swift.calls, expected_calls)
def test_mpu_GET_last_version(self):
req = swob.Request.blank('/bucket/mpu', headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()
})
status, headers, body = self.call_s3api(req)
self.assertEqual(status, '200 OK')
self.assertEqual(headers['x-amz-meta-user-notes'], 'version2')
self.assertEqual(headers['Content-Length'], '75')
expected_calls = [
('HEAD', '/v1/AUTH_test'),
('HEAD', '/v1/AUTH_test/bucket'),
('HEAD', '/v1/AUTH_test/\x00versions\x00bucket'),
('GET', '/v1/AUTH_test/bucket/mpu'),
('GET', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s' % (
~utils.Timestamp(self.version_ids[2])).normal),
('HEAD', '/v1/AUTH_test/bucket+segments'),
('GET', '/v1/AUTH_test/bucket+segments/mpu/X2/1'
'?multipart-manifest=get'),
('GET', '/v1/AUTH_test/bucket+segments/mpu/X2/2'
'?multipart-manifest=get'),
('GET', '/v1/AUTH_test/bucket+segments/mpu/X2/3'
'?multipart-manifest=get'),
('GET', '/v1/AUTH_test/bucket+segments/mpu/X2/4'
'?multipart-manifest=get'),
('GET', '/v1/AUTH_test/bucket+segments/mpu/X2/5'
'?multipart-manifest=get'),
]
if self.s3_acl:
# the pre-flight head on version marker get's symlinked; but I
# think maybe symlink makes metadata addative?
expected_calls = expected_calls[:3] + [
('HEAD', '/v1/AUTH_test/bucket/mpu'),
('HEAD', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00'
'%s' % (~utils.Timestamp(self.version_ids[2])).normal),
] + expected_calls[3:]
self.assertEqual(expected_calls, self.swift.calls)
def test_mpu_HEAD_last_version(self):
req = swob.Request.blank('/bucket/mpu', method='HEAD', headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()
})
status, headers, body = self.call_s3api(req)
self.assertEqual(status, '200 OK')
self.assertEqual(headers['x-amz-meta-user-notes'], 'version2')
self.assertEqual(headers['Content-Length'], '75')
self.assertEqual([
('HEAD', '/v1/AUTH_test'),
('HEAD', '/v1/AUTH_test/bucket'),
('HEAD', '/v1/AUTH_test/\x00versions\x00bucket'),
('HEAD', '/v1/AUTH_test/bucket/mpu'),
('HEAD', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s' % (
~utils.Timestamp(self.version_ids[2])).normal),
], self.swift.calls)
def test_mpu_HEAD_version(self):
req = swob.Request.blank('/bucket/mpu', method='HEAD', params={
'versionId': self.version_ids[1],
}, headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()
})
status, headers, body = self.call_s3api(req)
self.assertEqual(status, '200 OK')
self.assertEqual(headers['x-amz-meta-user-notes'], 'version1')
self.assertEqual(headers['Content-Length'], '50')
self.assertEqual(body, b'')
self.assertEqual([
('HEAD', '/v1/AUTH_test'),
('HEAD', '/v1/AUTH_test/bucket'),
('HEAD', '/v1/AUTH_test/\x00versions\x00bucket'),
('HEAD', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s'
'?version-id=%s' % (
(~utils.Timestamp(self.version_ids[1])).normal,
self.version_ids[1])),
], self.swift.calls)
def test_mpu_GET_version_part_num(self):
req = swob.Request.blank('/bucket/mpu', params={
'versionId': self.version_ids[2],
'partNumber': 5,
}, headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()
})
status, headers, body = self.call_s3api(req)
self.assertEqual(status, '206 Partial Content')
self.assertEqual(headers['x-amz-meta-user-notes'], 'version2')
self.assertEqual(headers['Content-Length'], '25')
self.assertEqual(body, b'e' * 25)
expected_calls = [
('HEAD', '/v1/AUTH_test'),
('HEAD', '/v1/AUTH_test/bucket'),
('HEAD', '/v1/AUTH_test/\x00versions\x00bucket'),
('GET', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s'
'?part-number=5&version-id=%s' % (
(~utils.Timestamp(self.version_ids[2])).normal,
self.version_ids[2])),
('HEAD', '/v1/AUTH_test/bucket+segments'),
('GET', '/v1/AUTH_test/bucket+segments/mpu/X2/5'
'?multipart-manifest=get'),
]
if self.s3_acl:
expected_calls.insert(3, (
'HEAD', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s'
'?version-id=%s' % (
(~utils.Timestamp(self.version_ids[2])).normal,
self.version_ids[2])
))
self.assertEqual(expected_calls, self.swift.calls)
def test_mpu_HEAD_version_part_num(self):
req = swob.Request.blank('/bucket/mpu', method='HEAD', params={
'versionId': self.version_ids[2],
'partNumber': 3,
}, headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()
})
status, headers, body = self.call_s3api(req)
self.assertEqual(status, '206 Partial Content')
self.assertEqual(headers['x-amz-meta-user-notes'], 'version2')
self.assertEqual(headers['Content-Length'], '15')
self.assertEqual(body, b'')
self.assertEqual(self.swift.calls, [
('HEAD', '/v1/AUTH_test'),
('HEAD', '/v1/AUTH_test/bucket'),
('HEAD', '/v1/AUTH_test/\x00versions\x00bucket'),
('HEAD', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s'
'?part-number=3&version-id=%s' % (
(~utils.Timestamp(self.version_ids[2])).normal,
self.version_ids[2])),
('GET', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s'
'?part-number=3&version-id=%s' % (
(~utils.Timestamp(self.version_ids[2])).normal,
self.version_ids[2])),
])
def test_mpu_GET_last_version_part_num(self):
req = swob.Request.blank('/bucket/mpu', params={
'partNumber': 4,
}, headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()
})
status, headers, body = self.call_s3api(req)
self.assertEqual(status, '206 Partial Content')
self.assertEqual(headers['x-amz-meta-user-notes'], 'version2')
self.assertEqual(headers['Content-Length'], '20')
self.assertEqual(body, b'd' * 20)
expected_calls = [
('HEAD', '/v1/AUTH_test'),
('HEAD', '/v1/AUTH_test/bucket'),
('HEAD', '/v1/AUTH_test/\x00versions\x00bucket'),
('GET', '/v1/AUTH_test/bucket/mpu?part-number=4'),
('GET', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s'
'?part-number=4' % (
~utils.Timestamp(self.version_ids[2])).normal),
('HEAD', '/v1/AUTH_test/bucket+segments'),
('GET', '/v1/AUTH_test/bucket+segments/mpu/X2/4'
'?multipart-manifest=get'),
]
if self.s3_acl:
expected_calls = expected_calls[:3] + [
('HEAD', '/v1/AUTH_test/bucket/mpu'),
('HEAD', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00'
'%s' % (~utils.Timestamp(self.version_ids[2])).normal),
] + expected_calls[3:]
self.assertEqual(expected_calls, self.swift.calls)
def test_mpu_HEAD_last_version_part_num(self):
req = swob.Request.blank('/bucket/mpu', method='HEAD', params={
'partNumber': 5,
}, headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()
})
status, headers, body = self.call_s3api(req)
self.assertEqual(status, '206 Partial Content')
self.assertEqual(headers['x-amz-meta-user-notes'], 'version2')
self.assertEqual(headers['Content-Length'], '25')
self.assertEqual(self.swift.calls, [
('HEAD', '/v1/AUTH_test'),
('HEAD', '/v1/AUTH_test/bucket'),
('HEAD', '/v1/AUTH_test/\x00versions\x00bucket'),
('HEAD', '/v1/AUTH_test/bucket/mpu?part-number=5'),
('HEAD', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s'
'?part-number=5' % (
~utils.Timestamp(self.version_ids[2])).normal),
('GET', '/v1/AUTH_test/bucket/mpu?part-number=5'),
('GET', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s'
'?part-number=5' % (
~utils.Timestamp(self.version_ids[2])).normal),
])
class TestVersionedMpuGETorHEADAcl(TestVersionedMpuGETorHEAD,
S3ApiTestCaseAcl):
def setUp(self):
super(TestVersionedMpuGETorHEADAcl, self).setUp()
object_headers = _gen_test_headers(
self.default_owner, self.grants, 'object')
for version_id in self.version_ids:
# s3acl would add the default object ACL on PUT to each version
version_path = '/v1/AUTH_test/\x00versions\x00bucket/' \
'\x00mpu\x00%s' % (~utils.Timestamp(version_id)).normal
self.swift.update_sticky_response_headers(
version_path, object_headers)
# this is used to flag insertion of expected HEAD pre-flight request of
# object ACLs
self.s3_acl = True

23
test/unit/common/middleware/s3api/test_multi_upload.py
View File

@ -685,6 +685,9 @@ class BaseS3ApiMultiUpload(object):
body='part object')
status, headers, body = self.call_s3api(req)
self.assertEqual(self._get_error_code(body), 'InvalidArgument')
self.assertEqual(self._get_error_message(body),
'Part number must be an integer between 1 and 10000, '
'inclusive')
# part number must be > 0
req = Request.blank('/bucket/object?partNumber=0&uploadId=X',
@ -694,6 +697,9 @@ class BaseS3ApiMultiUpload(object):
body='part object')
status, headers, body = self.call_s3api(req)
self.assertEqual(self._get_error_code(body), 'InvalidArgument')
self.assertEqual(self._get_error_message(body),
'Part number must be an integer between 1 and 10000, '
'inclusive')
# part number must be < 10001
req = Request.blank('/bucket/object?partNumber=10001&uploadId=X',
@ -703,6 +709,23 @@ class BaseS3ApiMultiUpload(object):
body='part object')
status, headers, body = self.call_s3api(req)
self.assertEqual(self._get_error_code(body), 'InvalidArgument')
self.assertEqual(self._get_error_message(body),
'Part number must be an integer between 1 and 10000, '
'inclusive')
with patch.object(self.s3api.conf, 'max_upload_part_num', 1000):
# part number must be < 1001
req = Request.blank(
'/bucket/object?partNumber=1001&uploadId=X',
environ={'REQUEST_METHOD': 'PUT'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()},
body='part object')
status, headers, body = self.call_s3api(req)
self.assertEqual(self._get_error_code(body), 'InvalidArgument')
self.assertEqual(self._get_error_message(body),
'Part number must be an integer between 1 and '
'1000, inclusive')
# without target bucket
req = Request.blank('/nobucket/object?partNumber=1&uploadId=X',

141
test/unit/common/middleware/s3api/test_obj.py
View File

@ -119,6 +119,12 @@ class BaseS3ApiObj(object):
if method == 'GET':
self.assertEqual(body, self.object_body)
def test_object_GET(self):
self._test_object_GETorHEAD('GET')
def test_object_HEAD(self):
self._test_object_GETorHEAD('HEAD')
def test_object_HEAD_error(self):
# HEAD does not return the body even an error response in the
# specifications of the REST API.
@ -331,8 +337,136 @@ class BaseS3ApiObj(object):
expected_status='429 Slow Down')
self.assertEqual(code, 'SlowDown')
def test_object_GET(self):
self._test_object_GETorHEAD('GET')
def _test_non_slo_object_GETorHEAD_part_num(self, method, part_number):
req = Request.blank('/bucket/object?partNumber=%s' % part_number,
environ={'REQUEST_METHOD': method},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '206')
self.assertEqual(headers['content-length'], '5')
self.assertTrue('content-range' in headers)
self.assertEqual(headers['content-range'], 'bytes 0-4/5')
self.assertEqual(headers['content-type'], 'text/html')
# we'll want this for logging
self._assert_policy_index(req.headers, headers,
self.bucket_policy_index)
self.assertEqual(headers['etag'],
'"%s"' % self.response_headers['etag'])
if method == 'GET':
self.assertEqual(body, self.object_body)
def test_non_slo_object_GET_part_num(self):
self._test_non_slo_object_GETorHEAD_part_num('GET', 1)
def test_non_slo_object_HEAD_part_num(self):
self._test_non_slo_object_GETorHEAD_part_num('HEAD', 1)
def _do_test_non_slo_object_part_num_not_satisfiable(self, method,
part_number):
req = Request.blank('/bucket/object',
params={'partNumber': part_number},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
req.method = method
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '416')
return body
def test_non_slo_object_GET_part_num_not_satisfiable(self):
body = self._do_test_non_slo_object_part_num_not_satisfiable(
'GET', '2')
self.assertEqual(self._get_error_code(body), 'InvalidPartNumber')
body = self._do_test_non_slo_object_part_num_not_satisfiable(
'GET', '10000')
self.assertEqual(self._get_error_code(body), 'InvalidPartNumber')
def test_non_slo_object_HEAD_part_num_not_satisfiable(self):
body = self._do_test_non_slo_object_part_num_not_satisfiable(
'HEAD', '2')
self.assertEqual(body, b'')
body = self._do_test_non_slo_object_part_num_not_satisfiable(
'HEAD', '10000')
self.assertEqual(body, b'')
def _do_test_non_slo_object_part_num_invalid(self, method, part_number):
req = Request.blank('/bucket/object',
params={'partNumber': part_number},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
req.method = method
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '400')
return body
def test_non_slo_object_GET_part_num_invalid(self):
body = self._do_test_non_slo_object_part_num_invalid('GET', '0')
self.assertEqual(self._get_error_code(body), 'InvalidArgument')
body = self._do_test_non_slo_object_part_num_invalid('GET', '-1')
self.assertEqual(self._get_error_code(body), 'InvalidArgument')
body = self._do_test_non_slo_object_part_num_invalid('GET', '10001')
self.assertEqual(self._get_error_code(body), 'InvalidArgument')
with patch.object(self.s3api.conf, 'max_upload_part_num', 1000):
body = self._do_test_non_slo_object_part_num_invalid('GET', '1001')
self.assertEqual(self._get_error_code(body), 'InvalidArgument')
self.assertEqual(
self._get_error_message(body),
'Part number must be an integer between 1 and 1000, inclusive')
body = self._do_test_non_slo_object_part_num_invalid('GET', 'foo')
self.assertEqual(self._get_error_code(body), 'InvalidArgument')
self.assertEqual(
self._get_error_message(body),
'Part number must be an integer between 1 and 10000, inclusive')
def test_non_slo_object_HEAD_part_num_invalid(self):
body = self._do_test_non_slo_object_part_num_invalid('HEAD', '0')
self.assertEqual(body, b'')
body = self._do_test_non_slo_object_part_num_invalid('HEAD', '-1')
self.assertEqual(body, b'')
body = self._do_test_non_slo_object_part_num_invalid('HEAD', '10001')
self.assertEqual(body, b'')
body = self._do_test_non_slo_object_part_num_invalid('HEAD', 'foo')
self.assertEqual(body, b'')
def test_non_slo_object_GET_part_num_and_range(self):
req = Request.blank('/bucket/object',
params={'partNumber': '1'},
headers={'Range': 'bytes=1-2',
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
req.method = 'GET'
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '400')
self.assertEqual(self._get_error_code(body), 'InvalidRequest')
self.assertEqual(
self._get_error_message(body),
'Cannot specify both Range header and partNumber query parameter')
# partNumber + Range error trumps bad partNumber
req = Request.blank('/bucket/object',
params={'partNumber': '0'},
headers={'Range': 'bytes=1-2',
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
req.method = 'GET'
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '400')
self.assertEqual(self._get_error_code(body), 'InvalidRequest')
self.assertEqual(
self._get_error_message(body),
'Cannot specify both Range header and partNumber query parameter')
def test_non_slo_object_HEAD_part_num_and_range(self):
req = Request.blank('/bucket/object',
params={'partNumber': '1'},
headers={'Range': 'bytes=1-2',
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
req.method = 'HEAD'
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '400')
def test_object_GET_Range(self):
req = Request.blank('/bucket/object',
@ -1105,9 +1239,6 @@ class TestS3ApiObj(BaseS3ApiObj, S3ApiTestCase):
swob.HTTPRequestedRangeNotSatisfiable)
self.assertEqual(code, 'InvalidRange')
def test_object_HEAD(self):
self._test_object_GETorHEAD('HEAD')
@patch_policies([
StoragePolicy(0, 'gold', is_default=True),
StoragePolicy(1, 'silver')])

138
test/unit/common/middleware/s3api/test_s3request.py
View File

@ -32,7 +32,8 @@ from swift.common.middleware.s3api.s3request import S3Request, \
S3AclRequest, SigV4Request, SIGV4_X_AMZ_DATE_FORMAT, HashingInput
from swift.common.middleware.s3api.s3response import InvalidArgument, \
NoSuchBucket, InternalError, ServiceUnavailable, \
AccessDenied, SignatureDoesNotMatch, RequestTimeTooSkewed, BadDigest
AccessDenied, SignatureDoesNotMatch, RequestTimeTooSkewed, BadDigest, \
InvalidPartArgument, InvalidPartNumber, InvalidRequest
from swift.common.utils import md5
from test.debug_logger import debug_logger
@ -998,6 +999,104 @@ class TestRequest(S3ApiTestCase):
sigv4_req._canonical_request().endswith(sha256_of_nothing.upper()))
self.assertTrue(sigv4_req.check_signature('secret'))
def test_validate_part_number(self):
sw_req = Request.blank('/nojunk',
environ={'REQUEST_METHOD': 'GET'},
headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
req = S3Request(sw_req.environ)
self.assertIsNone(req.validate_part_number())
# ok
sw_req = Request.blank('/nojunk?partNumber=102',
environ={'REQUEST_METHOD': 'GET'},
headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
req = S3Request(sw_req.environ)
self.assertEqual(102, req.validate_part_number())
req = S3Request(sw_req.environ,
conf=Config({'max_upload_part_num': 100}))
self.assertEqual(102, req.validate_part_number(102))
req = S3Request(sw_req.environ,
conf=Config({'max_upload_part_num': 102}))
self.assertEqual(102, req.validate_part_number(102))
def test_validate_part_number_invalid_argument(self):
def check_invalid_argument(part_num, max_parts, parts_count, exp_max):
sw_req = Request.blank('/nojunk?partNumber=%s' % part_num,
environ={'REQUEST_METHOD': 'GET'},
headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
req = S3Request(sw_req.environ,
conf=Config({'max_upload_part_num': max_parts}))
with self.assertRaises(InvalidPartArgument) as cm:
req.validate_part_number(parts_count=parts_count)
self.assertEqual('400 Bad Request', str(cm.exception))
self.assertIn(
b'Part number must be an integer between 1 and %d' % exp_max,
cm.exception.body)
check_invalid_argument(102, 99, None, 99)
check_invalid_argument(102, 100, 99, 100)
check_invalid_argument(102, 100, 101, 101)
check_invalid_argument(102, 101, 100, 101)
check_invalid_argument(102, 101, 101, 101)
check_invalid_argument('banana', 1000, None, 1000)
check_invalid_argument(0, 10000, None, 10000)
def test_validate_part_number_invalid_part_number(self):
def check_invalid_part_num(part_num, max_parts, parts_count):
sw_req = Request.blank('/nojunk?partNumber=%s' % part_num,
environ={'REQUEST_METHOD': 'GET'},
headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
req = S3Request(sw_req.environ,
conf=Config({'max_upload_part_num': max_parts}))
with self.assertRaises(InvalidPartNumber) as cm:
req.validate_part_number(parts_count=parts_count)
self.assertEqual('416 Requested Range Not Satisfiable',
str(cm.exception))
self.assertIn(b'The requested partnumber is not satisfiable',
cm.exception.body)
check_invalid_part_num(102, 10000, 1)
check_invalid_part_num(102, 102, 101)
check_invalid_part_num(102, 10000, 101)
def test_validate_part_number_with_range_header(self):
sw_req = Request.blank('/nojunk?partNumber=1',
environ={'REQUEST_METHOD': 'GET'},
headers={
'Range': 'bytes=1-2',
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
req = S3Request(sw_req.environ)
with self.assertRaises(InvalidRequest) as cm:
req.validate_part_number()
self.assertEqual('400 Bad Request',
str(cm.exception))
self.assertIn(b'Cannot specify both Range header and partNumber query '
b'parameter', cm.exception.body)
# bad part number AND Range header
sw_req = Request.blank('/nojunk?partNumber=0',
environ={'REQUEST_METHOD': 'GET'},
headers={
'Range': 'bytes=1-2',
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
req = S3Request(sw_req.environ)
with self.assertRaises(InvalidRequest) as cm:
req.validate_part_number()
self.assertEqual('400 Bad Request',
str(cm.exception))
self.assertIn(b'Cannot specify both Range header and partNumber query '
b'parameter', cm.exception.body)
class TestSigV4Request(S3ApiTestCase):
def setUp(self):
@ -1204,7 +1303,7 @@ class TestSigV4Request(S3ApiTestCase):
return SigV4Request(req.environ, None, config)
s3req = make_s3req(Config(), '/bkt', {'partNumber': '3'})
self.assertEqual(controllers.multi_upload.PartController,
self.assertEqual(controllers.ObjectController,
s3req.controller)
s3req = make_s3req(Config(), '/bkt', {'uploadId': '4'})
@ -1235,6 +1334,41 @@ class TestSigV4Request(S3ApiTestCase):
self.assertEqual(controllers.ServiceController,
s3req.controller)
def test_controller_for_multipart_upload_requests(self):
environ = {
'HTTP_HOST': 'bucket.s3.test.com',
'REQUEST_METHOD': 'PUT'}
x_amz_date = self.get_v4_amz_date_header()
auth = ('AWS4-HMAC-SHA256 '
'Credential=test/%s/us-east-1/s3/aws4_request,'
'SignedHeaders=host;x-amz-content-sha256;x-amz-date,'
'Signature=X' % self.get_v4_amz_date_header().split('T', 1)[0])
headers = {
'Authorization': auth,
'X-Amz-Content-SHA256': '0123456789',
'Date': self.get_date_header(),
'X-Amz-Date': x_amz_date}
def make_s3req(config, path, params):
req = Request.blank(path, environ=environ, headers=headers,
params=params)
return SigV4Request(req.environ, None, config)
s3req = make_s3req(Config(), '/bkt', {'partNumber': '3',
'uploadId': '4'})
self.assertEqual(controllers.multi_upload.PartController,
s3req.controller)
s3req = make_s3req(Config(), '/bkt', {'partNumber': '3'})
self.assertEqual(controllers.multi_upload.PartController,
s3req.controller)
s3req = make_s3req(Config(), '/bkt', {'uploadId': '4',
'partNumber': '3',
'copySource': 'bkt2/obj2'})
self.assertEqual(controllers.multi_upload.PartController,
s3req.controller)
class TestHashingInput(S3ApiTestCase):
def test_good(self):

1
test/unit/common/middleware/s3api/test_utils.py
View File

@ -181,6 +181,7 @@ class TestConfig(unittest.TestCase):
del conf.allow_no_owner
del conf.allowable_clock_skew
del conf.ratelimit_as_client_error
del conf.max_upload_part_num
self.assertEqual({}, conf)
def test_update(self):