Start transition to boto3 in s3api tests.
Adds the scaffolding required for tests to use boto3 and converts the test_bucket.py tests to the new interface. Follow on patches will convert the other tests to use the boto3 library. Notable changes: we no longer try to reach for the equivalent of `boto.make_request()` and instead rely on the boto3/botocore event system to mutate requests as necessary (or to disable pre-flight validators). Partial-Bug: 1557260 Change-Id: I3d77ef4a6b878c49ebfa0c8b8647d7199d87601e
This commit is contained in:
Timur Alperovich
Tim Burke
parent
2f05025255
commit
d4e7940457
@ -2,7 +2,8 @@
|
||||
# of appearance. Changing the order has an impact on the overall integration
|
||||
# process, which may cause wedges in the gate later.
|
||||
# this is required for the docs build jobs
|
||||
sphinx>=1.6.2 # BSD
|
||||
sphinx>=1.6.2,<2.0.0;python_version=='2.7' # BSD
|
||||
sphinx>=1.6.2;python_version>='3.4' # BSD
|
||||
openstackdocstheme>=1.11.0 # Apache-2.0
|
||||
reno>=1.8.0 # Apache-2.0
|
||||
os-api-ref>=1.0.0 # Apache-2.0
|
||||
|
||||
@ -3,6 +3,8 @@ asn1crypto==0.24.0
|
||||
Babel==2.5.3
|
||||
bandit==1.1.0
|
||||
boto==2.32.1
|
||||
boto3==1.9
|
||||
botocore==1.12
|
||||
castellan==0.13.0
|
||||
certifi==2018.1.18
|
||||
cffi==1.11.5
|
||||
|
||||
@ -15,6 +15,8 @@ python-keystoneclient!=2.1.0,>=2.0.0 # Apache-2.0
|
||||
reno>=1.8.0 # Apache-2.0
|
||||
python-openstackclient>=3.12.0
|
||||
boto>=2.32.1
|
||||
boto3>=1.9
|
||||
botocore>=1.12
|
||||
requests-mock>=1.2.0 # Apache-2.0
|
||||
fixtures>=3.0.0 # Apache-2.0/BSD
|
||||
keystonemiddleware>=4.17.0 # Apache-2.0
|
||||
|
||||
@ -16,7 +16,8 @@
|
||||
import unittest2
|
||||
import traceback
|
||||
import test.functional as tf
|
||||
from test.functional.s3api.s3_test_client import Connection
|
||||
from test.functional.s3api.s3_test_client import (
|
||||
Connection, get_boto3_conn, tear_down_s3)
|
||||
|
||||
|
||||
def setUpModule():
|
||||
@ -59,3 +60,23 @@ class S3ApiBase(unittest2.TestCase):
|
||||
if etag is not None:
|
||||
self.assertTrue('etag' in headers) # sanity
|
||||
self.assertEqual(etag, headers['etag'].strip('"'))
|
||||
|
||||
|
||||
class S3ApiBaseBoto3(S3ApiBase):
|
||||
def setUp(self):
|
||||
if 's3api' not in tf.cluster_info:
|
||||
raise tf.SkipTest('s3api middleware is not enabled')
|
||||
try:
|
||||
self.conn = get_boto3_conn()
|
||||
self.endpoint_url = self.conn._endpoint.host
|
||||
self.access_key = self.conn._request_signer._credentials.access_key
|
||||
self.region = self.conn._client_config.region_name
|
||||
tear_down_s3(self.conn)
|
||||
except Exception:
|
||||
message = '%s got an error during initialize process.\n\n%s' % \
|
||||
(self.method_name, traceback.format_exc())
|
||||
# TODO: Find a way to make this go to FAIL instead of Error
|
||||
self.fail(message)
|
||||
|
||||
def tearDown(self):
|
||||
tear_down_s3(self.conn)
|
||||
|
||||
@ -15,6 +15,8 @@
|
||||
|
||||
import os
|
||||
import test.functional as tf
|
||||
import boto3
|
||||
from botocore.exceptions import ClientError
|
||||
from boto.s3.connection import S3Connection, OrdinaryCallingFormat, \
|
||||
S3ResponseError
|
||||
import six
|
||||
@ -135,6 +137,53 @@ class Connection(object):
|
||||
return url, {}
|
||||
|
||||
|
||||
def get_boto3_conn(aws_access_key='test:tester', aws_secret_key='testing'):
|
||||
host = tf.config['auth_host']
|
||||
port = int(tf.config['auth_port'])
|
||||
config = boto3.session.Config(s3={'addressing_style': 'path'})
|
||||
return boto3.client(
|
||||
's3', aws_access_key_id=aws_access_key,
|
||||
aws_secret_access_key=aws_secret_key,
|
||||
config=config, region_name='us-east-1', use_ssl=False,
|
||||
endpoint_url='http://{}:{}'.format(host, port))
|
||||
|
||||
|
||||
def tear_down_s3(conn):
|
||||
"""
|
||||
Reset all swift environment to keep clean. As a result by calling this
|
||||
method, we can assume the backend swift keeps no containers and no
|
||||
objects on this connection's account.
|
||||
"""
|
||||
exceptions = []
|
||||
for i in range(RETRY_COUNT):
|
||||
try:
|
||||
resp = conn.list_buckets()
|
||||
buckets = [bucket['Name'] for bucket in resp.get('Buckets', [])]
|
||||
for bucket in buckets:
|
||||
try:
|
||||
resp = conn.list_multipart_uploads(Bucket=bucket)
|
||||
for upload in resp.get('Uploads', []):
|
||||
conn.abort_multipart_upload(
|
||||
Bucket=bucket,
|
||||
Key=upload['Key'],
|
||||
UploadId=upload['UploadId'])
|
||||
|
||||
resp = conn.list_objects(Bucket=bucket)
|
||||
for obj in resp.get('Contents', []):
|
||||
conn.delete_object(Bucket=bucket, Key=obj['Key'])
|
||||
conn.delete_bucket(Bucket=bucket)
|
||||
except ClientError as e:
|
||||
# 404 means NoSuchBucket, NoSuchKey, or NoSuchUpload
|
||||
if e.response['ResponseMetadata']['HTTPStatusCode'] != 404:
|
||||
raise
|
||||
except Exception as e:
|
||||
exceptions.append(''.join(
|
||||
traceback.format_exception(*sys.exc_info())))
|
||||
if exceptions:
|
||||
exceptions.insert(0, 'Too many errors to continue:')
|
||||
raise Exception('\n========\n'.join(exceptions))
|
||||
|
||||
|
||||
# TODO: make sure where this function is used
|
||||
def get_admin_connection():
|
||||
"""
|
||||
|
||||
@ -13,16 +13,15 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import botocore
|
||||
import datetime
|
||||
import unittest2
|
||||
import os
|
||||
|
||||
import test.functional as tf
|
||||
from swift.common.utils import config_true_value
|
||||
from swift.common.middleware.s3api.etree import fromstring, tostring, Element, \
|
||||
SubElement
|
||||
from test.functional.s3api import S3ApiBase
|
||||
from test.functional.s3api.s3_test_client import Connection
|
||||
from test.functional.s3api.utils import get_error_code
|
||||
from test.functional.s3api import S3ApiBaseBoto3
|
||||
from test.functional.s3api.s3_test_client import get_boto3_conn
|
||||
|
||||
|
||||
def setUpModule():
|
||||
@ -33,14 +32,21 @@ def tearDownModule():
|
||||
tf.teardown_package()
|
||||
|
||||
|
||||
class TestS3ApiBucket(S3ApiBase):
|
||||
def setUp(self):
|
||||
super(TestS3ApiBucket, self).setUp()
|
||||
|
||||
def _gen_location_xml(self, location):
|
||||
elem = Element('CreateBucketConfiguration')
|
||||
SubElement(elem, 'LocationConstraint').text = location
|
||||
return tostring(elem)
|
||||
class TestS3ApiBucket(S3ApiBaseBoto3):
|
||||
def _validate_object_listing(self, resp_objects, req_objects,
|
||||
expect_owner=True):
|
||||
self.assertEqual(len(resp_objects), len(req_objects))
|
||||
for i, obj in enumerate(resp_objects):
|
||||
self.assertEqual(obj['Key'], req_objects[i])
|
||||
self.assertEqual(type(obj['LastModified']), datetime.datetime)
|
||||
self.assertIn('ETag', obj)
|
||||
self.assertIn('Size', obj)
|
||||
self.assertEqual(obj['StorageClass'], 'STANDARD')
|
||||
if expect_owner:
|
||||
self.assertEqual(obj['Owner']['ID'], self.access_key)
|
||||
self.assertEqual(obj['Owner']['DisplayName'], self.access_key)
|
||||
else:
|
||||
self.assertNotIn('Owner', obj)
|
||||
|
||||
def test_bucket(self):
|
||||
bucket = 'bucket'
|
||||
@ -48,112 +54,95 @@ class TestS3ApiBucket(S3ApiBase):
|
||||
'max_bucket_listing', 1000)
|
||||
|
||||
# PUT Bucket
|
||||
status, headers, body = self.conn.make_request('PUT', bucket)
|
||||
self.assertEqual(status, 200)
|
||||
resp = self.conn.create_bucket(Bucket=bucket)
|
||||
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
|
||||
headers = resp['ResponseMetadata']['HTTPHeaders']
|
||||
|
||||
self.assertCommonResponseHeaders(headers)
|
||||
self.assertIn(headers['location'], (
|
||||
'/' + bucket, # swob won't touch it...
|
||||
# but webob (which we get because of auth_token) *does*
|
||||
'http://%s%s/%s' % (
|
||||
self.conn.host,
|
||||
'' if self.conn.port == 80 else ':%d' % self.conn.port,
|
||||
bucket),
|
||||
# This is all based on the Host header the client provided,
|
||||
# and boto will double-up ports for sig v4. See
|
||||
# - https://github.com/boto/boto/issues/2623
|
||||
# - https://github.com/boto/boto/issues/3716
|
||||
# with proposed fixes at
|
||||
# - https://github.com/boto/boto/pull/3513
|
||||
# - https://github.com/boto/boto/pull/3676
|
||||
'http://%s%s:%d/%s' % (
|
||||
self.conn.host,
|
||||
'' if self.conn.port == 80 else ':%d' % self.conn.port,
|
||||
self.conn.port,
|
||||
bucket),
|
||||
'%s/%s' % (self.endpoint_url, bucket),
|
||||
))
|
||||
self.assertEqual(headers['content-length'], '0')
|
||||
|
||||
# GET Bucket(Without Object)
|
||||
status, headers, body = self.conn.make_request('GET', bucket)
|
||||
self.assertEqual(status, 200)
|
||||
resp = self.conn.list_objects(Bucket=bucket)
|
||||
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
|
||||
headers = resp['ResponseMetadata']['HTTPHeaders']
|
||||
|
||||
self.assertCommonResponseHeaders(headers)
|
||||
self.assertIsNotNone(headers['content-type'])
|
||||
self.assertEqual(headers['content-length'], str(len(body)))
|
||||
# TODO; requires consideration
|
||||
# self.assertEqual(headers['transfer-encoding'], 'chunked')
|
||||
|
||||
elem = fromstring(body, 'ListBucketResult')
|
||||
self.assertEqual(elem.find('Name').text, bucket)
|
||||
self.assertIsNone(elem.find('Prefix').text)
|
||||
self.assertIsNone(elem.find('Marker').text)
|
||||
self.assertEqual(
|
||||
elem.find('MaxKeys').text, str(max_bucket_listing))
|
||||
self.assertEqual(elem.find('IsTruncated').text, 'false')
|
||||
objects = elem.findall('./Contents')
|
||||
self.assertEqual(list(objects), [])
|
||||
self.assertEqual(resp['Name'], bucket)
|
||||
self.assertEqual(resp['Prefix'], '')
|
||||
self.assertEqual(resp['Marker'], '')
|
||||
self.assertEqual(resp['MaxKeys'], max_bucket_listing)
|
||||
self.assertFalse(resp['IsTruncated'])
|
||||
self.assertNotIn('Contents', bucket)
|
||||
|
||||
# GET Bucket(With Object)
|
||||
req_objects = ('object', 'object2')
|
||||
req_objects = ['object', 'object2']
|
||||
for obj in req_objects:
|
||||
self.conn.make_request('PUT', bucket, obj)
|
||||
status, headers, body = self.conn.make_request('GET', bucket)
|
||||
self.assertEqual(status, 200)
|
||||
self.conn.put_object(Bucket=bucket, Key=obj, Body=b'')
|
||||
resp = self.conn.list_objects(Bucket=bucket)
|
||||
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
|
||||
|
||||
elem = fromstring(body, 'ListBucketResult')
|
||||
self.assertEqual(elem.find('Name').text, bucket)
|
||||
self.assertIsNone(elem.find('Prefix').text)
|
||||
self.assertIsNone(elem.find('Marker').text)
|
||||
self.assertEqual(elem.find('MaxKeys').text,
|
||||
str(max_bucket_listing))
|
||||
self.assertEqual(elem.find('IsTruncated').text, 'false')
|
||||
resp_objects = elem.findall('./Contents')
|
||||
self.assertEqual(len(list(resp_objects)), 2)
|
||||
for o in resp_objects:
|
||||
self.assertIn(o.find('Key').text, req_objects)
|
||||
self.assertIsNotNone(o.find('LastModified').text)
|
||||
self.assertRegexpMatches(
|
||||
o.find('LastModified').text,
|
||||
r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
|
||||
self.assertIsNotNone(o.find('ETag').text)
|
||||
self.assertIsNotNone(o.find('Size').text)
|
||||
self.assertIsNotNone(o.find('StorageClass').text)
|
||||
self.assertEqual(o.find('Owner/ID').text, self.conn.user_id)
|
||||
self.assertEqual(o.find('Owner/DisplayName').text,
|
||||
self.conn.user_id)
|
||||
self.assertEqual(resp['Name'], bucket)
|
||||
self.assertEqual(resp['Prefix'], '')
|
||||
self.assertEqual(resp['Marker'], '')
|
||||
self.assertEqual(resp['MaxKeys'], max_bucket_listing)
|
||||
self.assertFalse(resp['IsTruncated'])
|
||||
self._validate_object_listing(resp['Contents'], req_objects)
|
||||
|
||||
# HEAD Bucket
|
||||
status, headers, body = self.conn.make_request('HEAD', bucket)
|
||||
self.assertEqual(status, 200)
|
||||
resp = self.conn.head_bucket(Bucket=bucket)
|
||||
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
|
||||
headers = resp['ResponseMetadata']['HTTPHeaders']
|
||||
|
||||
self.assertCommonResponseHeaders(headers)
|
||||
self.assertIsNotNone(headers['content-type'])
|
||||
self.assertEqual(headers['content-length'], str(len(body)))
|
||||
# TODO; requires consideration
|
||||
# self.assertEqual(headers['transfer-encoding'], 'chunked')
|
||||
|
||||
# DELETE Bucket
|
||||
for obj in req_objects:
|
||||
self.conn.make_request('DELETE', bucket, obj)
|
||||
status, headers, body = self.conn.make_request('DELETE', bucket)
|
||||
self.assertEqual(status, 204)
|
||||
self.conn.delete_object(Bucket=bucket, Key=obj)
|
||||
resp = self.conn.delete_bucket(Bucket=bucket)
|
||||
self.assertEqual(204, resp['ResponseMetadata']['HTTPStatusCode'])
|
||||
|
||||
self.assertCommonResponseHeaders(headers)
|
||||
self.assertCommonResponseHeaders(
|
||||
resp['ResponseMetadata']['HTTPHeaders'])
|
||||
|
||||
def test_put_bucket_error(self):
|
||||
status, headers, body = \
|
||||
self.conn.make_request('PUT', 'bucket+invalid')
|
||||
self.assertEqual(get_error_code(body), 'InvalidBucketName')
|
||||
event_system = self.conn.meta.events
|
||||
event_system.unregister(
|
||||
'before-parameter-build.s3',
|
||||
botocore.handlers.validate_bucket_name)
|
||||
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
|
||||
self.conn.create_bucket(Bucket='bucket+invalid')
|
||||
self.assertEqual(
|
||||
ctx.exception.response['ResponseMetadata']['HTTPStatusCode'], 400)
|
||||
self.assertEqual(
|
||||
ctx.exception.response['Error']['Code'], 'InvalidBucketName')
|
||||
|
||||
auth_error_conn = Connection(aws_secret_key='invalid')
|
||||
status, headers, body = auth_error_conn.make_request('PUT', 'bucket')
|
||||
self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
|
||||
auth_error_conn = get_boto3_conn(aws_secret_key='invalid')
|
||||
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
|
||||
auth_error_conn.create_bucket(Bucket='bucket')
|
||||
self.assertEqual(
|
||||
ctx.exception.response['ResponseMetadata']['HTTPStatusCode'], 403)
|
||||
self.assertEqual(ctx.exception.response['Error']['Code'],
|
||||
'SignatureDoesNotMatch')
|
||||
|
||||
self.conn.make_request('PUT', 'bucket')
|
||||
status, headers, body = self.conn.make_request('PUT', 'bucket')
|
||||
self.assertEqual(status, 409)
|
||||
self.assertEqual(get_error_code(body), 'BucketAlreadyOwnedByYou')
|
||||
self.conn.create_bucket(Bucket='bucket')
|
||||
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
|
||||
self.conn.create_bucket(Bucket='bucket')
|
||||
self.assertEqual(
|
||||
ctx.exception.response['ResponseMetadata']['HTTPStatusCode'], 409)
|
||||
self.assertEqual(
|
||||
ctx.exception.response['Error']['Code'], 'BucketAlreadyOwnedByYou')
|
||||
|
||||
def test_put_bucket_error_key2(self):
|
||||
if config_true_value(tf.cluster_info['s3api'].get('s3_acl')):
|
||||
@ -163,15 +152,18 @@ class TestS3ApiBucket(S3ApiBase):
|
||||
'Cannot test for BucketAlreadyExists with second user; '
|
||||
'need s3_access_key2 and s3_secret_key2 configured')
|
||||
|
||||
self.conn.make_request('PUT', 'bucket')
|
||||
self.conn.create_bucket(Bucket='bucket')
|
||||
|
||||
# Other users of the same account get the same 409 error
|
||||
conn2 = Connection(tf.config['s3_access_key2'],
|
||||
tf.config['s3_secret_key2'],
|
||||
tf.config['s3_access_key2'])
|
||||
status, headers, body = conn2.make_request('PUT', 'bucket')
|
||||
self.assertEqual(status, 409)
|
||||
self.assertEqual(get_error_code(body), 'BucketAlreadyExists')
|
||||
conn2 = get_boto3_conn(tf.config['s3_access_key2'],
|
||||
tf.config['s3_secret_key2'])
|
||||
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
|
||||
conn2.create_bucket(Bucket='bucket')
|
||||
self.assertEqual(
|
||||
ctx.exception.response['ResponseMetadata']['HTTPStatusCode'],
|
||||
409)
|
||||
self.assertEqual(
|
||||
ctx.exception.response['Error']['Code'], 'BucketAlreadyExists')
|
||||
|
||||
def test_put_bucket_error_key3(self):
|
||||
if 's3_access_key3' not in tf.config or \
|
||||
@ -179,41 +171,51 @@ class TestS3ApiBucket(S3ApiBase):
|
||||
raise tf.SkipTest('Cannot test for AccessDenied; need '
|
||||
's3_access_key3 and s3_secret_key3 configured')
|
||||
|
||||
self.conn.make_request('PUT', 'bucket')
|
||||
self.conn.create_bucket(Bucket='bucket')
|
||||
# If the user can't create buckets, they shouldn't even know
|
||||
# whether the bucket exists.
|
||||
conn3 = Connection(tf.config['s3_access_key3'],
|
||||
tf.config['s3_secret_key3'],
|
||||
tf.config['s3_access_key3'])
|
||||
status, headers, body = conn3.make_request('PUT', 'bucket')
|
||||
self.assertEqual(status, 403)
|
||||
self.assertEqual(get_error_code(body), 'AccessDenied')
|
||||
conn3 = get_boto3_conn(tf.config['s3_access_key3'],
|
||||
tf.config['s3_secret_key3'])
|
||||
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
|
||||
conn3.create_bucket(Bucket='bucket')
|
||||
self.assertEqual(
|
||||
ctx.exception.response['ResponseMetadata']['HTTPStatusCode'], 403)
|
||||
self.assertEqual(
|
||||
ctx.exception.response['Error']['Code'], 'AccessDenied')
|
||||
|
||||
def test_put_bucket_with_LocationConstraint(self):
|
||||
bucket = 'bucket'
|
||||
xml = self._gen_location_xml(self.conn.conn.auth_region_name)
|
||||
status, headers, body = \
|
||||
self.conn.make_request('PUT', bucket, body=xml)
|
||||
self.assertEqual(status, 200)
|
||||
resp = self.conn.create_bucket(
|
||||
Bucket='bucket',
|
||||
CreateBucketConfiguration={'LocationConstraint': self.region})
|
||||
self.assertEqual(resp['ResponseMetadata']['HTTPStatusCode'], 200)
|
||||
|
||||
def test_get_bucket_error(self):
|
||||
self.conn.make_request('PUT', 'bucket')
|
||||
event_system = self.conn.meta.events
|
||||
event_system.unregister(
|
||||
'before-parameter-build.s3',
|
||||
botocore.handlers.validate_bucket_name)
|
||||
self.conn.create_bucket(Bucket='bucket')
|
||||
|
||||
status, headers, body = \
|
||||
self.conn.make_request('GET', 'bucket+invalid')
|
||||
self.assertEqual(get_error_code(body), 'InvalidBucketName')
|
||||
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
|
||||
self.conn.list_objects(Bucket='bucket+invalid')
|
||||
self.assertEqual(
|
||||
ctx.exception.response['Error']['Code'], 'InvalidBucketName')
|
||||
|
||||
auth_error_conn = Connection(aws_secret_key='invalid')
|
||||
status, headers, body = auth_error_conn.make_request('GET', 'bucket')
|
||||
self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
|
||||
auth_error_conn = get_boto3_conn(aws_secret_key='invalid')
|
||||
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
|
||||
auth_error_conn.list_objects(Bucket='bucket')
|
||||
self.assertEqual(
|
||||
ctx.exception.response['Error']['Code'], 'SignatureDoesNotMatch')
|
||||
|
||||
status, headers, body = self.conn.make_request('GET', 'nothing')
|
||||
self.assertEqual(get_error_code(body), 'NoSuchBucket')
|
||||
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
|
||||
self.conn.list_objects(Bucket='nothing')
|
||||
self.assertEqual(
|
||||
ctx.exception.response['Error']['Code'], 'NoSuchBucket')
|
||||
|
||||
def _prepare_test_get_bucket(self, bucket, objects):
|
||||
self.conn.make_request('PUT', bucket)
|
||||
self.conn.create_bucket(Bucket=bucket)
|
||||
for obj in objects:
|
||||
self.conn.make_request('PUT', bucket, obj)
|
||||
self.conn.put_object(Bucket=bucket, Key=obj, Body=b'')
|
||||
|
||||
def test_get_bucket_with_delimiter(self):
|
||||
bucket = 'bucket'
|
||||
@ -222,32 +224,16 @@ class TestS3ApiBucket(S3ApiBase):
|
||||
self._prepare_test_get_bucket(bucket, put_objects)
|
||||
|
||||
delimiter = '/'
|
||||
query = 'delimiter=%s' % delimiter
|
||||
expect_objects = ('object', 'object2')
|
||||
expect_prefixes = ('dir/', 'subdir/', 'subdir2/')
|
||||
status, headers, body = \
|
||||
self.conn.make_request('GET', bucket, query=query)
|
||||
self.assertEqual(status, 200)
|
||||
elem = fromstring(body, 'ListBucketResult')
|
||||
self.assertEqual(elem.find('Delimiter').text, delimiter)
|
||||
resp_objects = elem.findall('./Contents')
|
||||
self.assertEqual(len(list(resp_objects)), len(expect_objects))
|
||||
for i, o in enumerate(resp_objects):
|
||||
self.assertEqual(o.find('Key').text, expect_objects[i])
|
||||
self.assertIsNotNone(o.find('LastModified').text)
|
||||
self.assertRegexpMatches(
|
||||
o.find('LastModified').text,
|
||||
r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
|
||||
self.assertIsNotNone(o.find('ETag').text)
|
||||
self.assertIsNotNone(o.find('Size').text)
|
||||
self.assertEqual(o.find('StorageClass').text, 'STANDARD')
|
||||
self.assertEqual(o.find('Owner/ID').text, self.conn.user_id)
|
||||
self.assertEqual(o.find('Owner/DisplayName').text,
|
||||
self.conn.user_id)
|
||||
resp_prefixes = elem.findall('CommonPrefixes')
|
||||
self.assertEqual(len(resp_prefixes), len(expect_prefixes))
|
||||
for i, p in enumerate(resp_prefixes):
|
||||
self.assertEqual(p.find('./Prefix').text, expect_prefixes[i])
|
||||
resp = self.conn.list_objects(Bucket=bucket, Delimiter=delimiter)
|
||||
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
|
||||
self.assertEqual(resp['Delimiter'], delimiter)
|
||||
self._validate_object_listing(resp['Contents'], expect_objects)
|
||||
resp_prefixes = resp['CommonPrefixes']
|
||||
self.assertEqual(
|
||||
resp_prefixes,
|
||||
[{'Prefix': p} for p in expect_prefixes])
|
||||
|
||||
def test_get_bucket_with_encoding_type(self):
|
||||
bucket = 'bucket'
|
||||
@ -255,12 +241,10 @@ class TestS3ApiBucket(S3ApiBase):
|
||||
self._prepare_test_get_bucket(bucket, put_objects)
|
||||
|
||||
encoding_type = 'url'
|
||||
query = 'encoding-type=%s' % encoding_type
|
||||
status, headers, body = \
|
||||
self.conn.make_request('GET', bucket, query=query)
|
||||
self.assertEqual(status, 200)
|
||||
elem = fromstring(body, 'ListBucketResult')
|
||||
self.assertEqual(elem.find('EncodingType').text, encoding_type)
|
||||
resp = self.conn.list_objects(
|
||||
Bucket=bucket, EncodingType=encoding_type)
|
||||
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
|
||||
self.assertEqual(resp['EncodingType'], encoding_type)
|
||||
|
||||
def test_get_bucket_with_marker(self):
|
||||
bucket = 'bucket'
|
||||
@ -269,27 +253,11 @@ class TestS3ApiBucket(S3ApiBase):
|
||||
self._prepare_test_get_bucket(bucket, put_objects)
|
||||
|
||||
marker = 'object'
|
||||
query = 'marker=%s' % marker
|
||||
expect_objects = ('object2', 'subdir/object', 'subdir2/object')
|
||||
status, headers, body = \
|
||||
self.conn.make_request('GET', bucket, query=query)
|
||||
self.assertEqual(status, 200)
|
||||
elem = fromstring(body, 'ListBucketResult')
|
||||
self.assertEqual(elem.find('Marker').text, marker)
|
||||
resp_objects = elem.findall('./Contents')
|
||||
self.assertEqual(len(list(resp_objects)), len(expect_objects))
|
||||
for i, o in enumerate(resp_objects):
|
||||
self.assertEqual(o.find('Key').text, expect_objects[i])
|
||||
self.assertIsNotNone(o.find('LastModified').text)
|
||||
self.assertRegexpMatches(
|
||||
o.find('LastModified').text,
|
||||
r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
|
||||
self.assertIsNotNone(o.find('ETag').text)
|
||||
self.assertIsNotNone(o.find('Size').text)
|
||||
self.assertEqual(o.find('StorageClass').text, 'STANDARD')
|
||||
self.assertEqual(o.find('Owner/ID').text, self.conn.user_id)
|
||||
self.assertEqual(o.find('Owner/DisplayName').text,
|
||||
self.conn.user_id)
|
||||
resp = self.conn.list_objects(Bucket=bucket, Marker=marker)
|
||||
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
|
||||
self.assertEqual(resp['Marker'], marker)
|
||||
self._validate_object_listing(resp['Contents'], expect_objects)
|
||||
|
||||
def test_get_bucket_with_max_keys(self):
|
||||
bucket = 'bucket'
|
||||
@ -297,28 +265,12 @@ class TestS3ApiBucket(S3ApiBase):
|
||||
'dir/subdir/object')
|
||||
self._prepare_test_get_bucket(bucket, put_objects)
|
||||
|
||||
max_keys = '2'
|
||||
query = 'max-keys=%s' % max_keys
|
||||
max_keys = 2
|
||||
expect_objects = ('dir/subdir/object', 'object')
|
||||
status, headers, body = \
|
||||
self.conn.make_request('GET', bucket, query=query)
|
||||
self.assertEqual(status, 200)
|
||||
elem = fromstring(body, 'ListBucketResult')
|
||||
self.assertEqual(elem.find('MaxKeys').text, max_keys)
|
||||
resp_objects = elem.findall('./Contents')
|
||||
self.assertEqual(len(list(resp_objects)), len(expect_objects))
|
||||
for i, o in enumerate(resp_objects):
|
||||
self.assertEqual(o.find('Key').text, expect_objects[i])
|
||||
self.assertIsNotNone(o.find('LastModified').text)
|
||||
self.assertRegexpMatches(
|
||||
o.find('LastModified').text,
|
||||
r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
|
||||
self.assertIsNotNone(o.find('ETag').text)
|
||||
self.assertIsNotNone(o.find('Size').text)
|
||||
self.assertEqual(o.find('StorageClass').text, 'STANDARD')
|
||||
self.assertEqual(o.find('Owner/ID').text, self.conn.user_id)
|
||||
self.assertEqual(o.find('Owner/DisplayName').text,
|
||||
self.conn.user_id)
|
||||
resp = self.conn.list_objects(Bucket=bucket, MaxKeys=max_keys)
|
||||
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
|
||||
self.assertEqual(resp['MaxKeys'], max_keys)
|
||||
self._validate_object_listing(resp['Contents'], expect_objects)
|
||||
|
||||
def test_get_bucket_with_prefix(self):
|
||||
bucket = 'bucket'
|
||||
@ -327,27 +279,11 @@ class TestS3ApiBucket(S3ApiBase):
|
||||
self._prepare_test_get_bucket(bucket, req_objects)
|
||||
|
||||
prefix = 'object'
|
||||
query = 'prefix=%s' % prefix
|
||||
expect_objects = ('object', 'object2')
|
||||
status, headers, body = \
|
||||
self.conn.make_request('GET', bucket, query=query)
|
||||
self.assertEqual(status, 200)
|
||||
elem = fromstring(body, 'ListBucketResult')
|
||||
self.assertEqual(elem.find('Prefix').text, prefix)
|
||||
resp_objects = elem.findall('./Contents')
|
||||
self.assertEqual(len(list(resp_objects)), len(expect_objects))
|
||||
for i, o in enumerate(resp_objects):
|
||||
self.assertEqual(o.find('Key').text, expect_objects[i])
|
||||
self.assertIsNotNone(o.find('LastModified').text)
|
||||
self.assertRegexpMatches(
|
||||
o.find('LastModified').text,
|
||||
r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
|
||||
self.assertIsNotNone(o.find('ETag').text)
|
||||
self.assertIsNotNone(o.find('Size').text)
|
||||
self.assertEqual(o.find('StorageClass').text, 'STANDARD')
|
||||
self.assertEqual(o.find('Owner/ID').text, self.conn.user_id)
|
||||
self.assertEqual(o.find('Owner/DisplayName').text,
|
||||
self.conn.user_id)
|
||||
resp = self.conn.list_objects(Bucket=bucket, Prefix=prefix)
|
||||
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
|
||||
self.assertEqual(resp['Prefix'], prefix)
|
||||
self._validate_object_listing(resp['Contents'], expect_objects)
|
||||
|
||||
def test_get_bucket_v2_with_start_after(self):
|
||||
bucket = 'bucket'
|
||||
@ -356,26 +292,13 @@ class TestS3ApiBucket(S3ApiBase):
|
||||
self._prepare_test_get_bucket(bucket, put_objects)
|
||||
|
||||
marker = 'object'
|
||||
query = 'list-type=2&start-after=%s' % marker
|
||||
expect_objects = ('object2', 'subdir/object', 'subdir2/object')
|
||||
status, headers, body = \
|
||||
self.conn.make_request('GET', bucket, query=query)
|
||||
self.assertEqual(status, 200)
|
||||
elem = fromstring(body, 'ListBucketResult')
|
||||
self.assertEqual(elem.find('StartAfter').text, marker)
|
||||
resp_objects = elem.findall('./Contents')
|
||||
self.assertEqual(len(list(resp_objects)), len(expect_objects))
|
||||
for i, o in enumerate(resp_objects):
|
||||
self.assertEqual(o.find('Key').text, expect_objects[i])
|
||||
self.assertTrue(o.find('LastModified').text is not None)
|
||||
self.assertRegexpMatches(
|
||||
o.find('LastModified').text,
|
||||
r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
|
||||
self.assertTrue(o.find('ETag').text is not None)
|
||||
self.assertTrue(o.find('Size').text is not None)
|
||||
self.assertEqual(o.find('StorageClass').text, 'STANDARD')
|
||||
self.assertIsNone(o.find('Owner/ID'))
|
||||
self.assertIsNone(o.find('Owner/DisplayName'))
|
||||
resp = self.conn.list_objects_v2(Bucket=bucket, StartAfter=marker)
|
||||
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
|
||||
self.assertEqual(resp['StartAfter'], marker)
|
||||
self.assertEqual(resp['KeyCount'], 3)
|
||||
self._validate_object_listing(resp['Contents'], expect_objects,
|
||||
expect_owner=False)
|
||||
|
||||
def test_get_bucket_v2_with_fetch_owner(self):
|
||||
bucket = 'bucket'
|
||||
@ -383,28 +306,12 @@ class TestS3ApiBucket(S3ApiBase):
|
||||
'dir/subdir/object')
|
||||
self._prepare_test_get_bucket(bucket, put_objects)
|
||||
|
||||
query = 'list-type=2&fetch-owner=true'
|
||||
expect_objects = ('dir/subdir/object', 'object', 'object2',
|
||||
'subdir/object', 'subdir2/object')
|
||||
status, headers, body = \
|
||||
self.conn.make_request('GET', bucket, query=query)
|
||||
self.assertEqual(status, 200)
|
||||
elem = fromstring(body, 'ListBucketResult')
|
||||
self.assertEqual(elem.find('KeyCount').text, '5')
|
||||
resp_objects = elem.findall('./Contents')
|
||||
self.assertEqual(len(list(resp_objects)), len(expect_objects))
|
||||
for i, o in enumerate(resp_objects):
|
||||
self.assertEqual(o.find('Key').text, expect_objects[i])
|
||||
self.assertTrue(o.find('LastModified').text is not None)
|
||||
self.assertRegexpMatches(
|
||||
o.find('LastModified').text,
|
||||
r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
|
||||
self.assertTrue(o.find('ETag').text is not None)
|
||||
self.assertTrue(o.find('Size').text is not None)
|
||||
self.assertEqual(o.find('StorageClass').text, 'STANDARD')
|
||||
self.assertTrue(o.find('Owner/ID').text, self.conn.user_id)
|
||||
self.assertTrue(o.find('Owner/DisplayName').text,
|
||||
self.conn.user_id)
|
||||
resp = self.conn.list_objects_v2(Bucket=bucket, FetchOwner=True)
|
||||
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
|
||||
self.assertEqual(resp['KeyCount'], 5)
|
||||
self._validate_object_listing(resp['Contents'], expect_objects)
|
||||
|
||||
def test_get_bucket_v2_with_continuation_token_and_delimiter(self):
|
||||
bucket = 'bucket'
|
||||
@ -421,86 +328,116 @@ class TestS3ApiBucket(S3ApiBase):
|
||||
'subdirs': []}]
|
||||
|
||||
continuation_token = ''
|
||||
query = 'list-type=2&max-keys=3&delimiter=/&continuation-token=%s'
|
||||
|
||||
for i in range(len(expected)):
|
||||
status, headers, body = self.conn.make_request(
|
||||
'GET', bucket, query=query % continuation_token)
|
||||
self.assertEqual(status, 200)
|
||||
elem = fromstring(body, 'ListBucketResult')
|
||||
self.assertEqual(elem.find('MaxKeys').text, '3')
|
||||
resp = self.conn.list_objects_v2(
|
||||
Bucket=bucket,
|
||||
MaxKeys=3,
|
||||
Delimiter='/',
|
||||
ContinuationToken=continuation_token)
|
||||
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
|
||||
self.assertEqual(resp['MaxKeys'], 3)
|
||||
self.assertEqual(
|
||||
elem.find('KeyCount').text,
|
||||
str(len(expected[i]['objects']) + len(expected[i]['subdirs'])))
|
||||
expect_truncated = 'true' if i < len(expected) - 1 else 'false'
|
||||
self.assertEqual(elem.find('IsTruncated').text, expect_truncated)
|
||||
next_cont_token_elem = elem.find('NextContinuationToken')
|
||||
if expect_truncated == 'true':
|
||||
self.assertIsNotNone(next_cont_token_elem)
|
||||
continuation_token = next_cont_token_elem.text
|
||||
resp_objects = elem.findall('./Contents')
|
||||
resp['KeyCount'],
|
||||
len(expected[i]['objects']) + len(expected[i]['subdirs']))
|
||||
expect_truncated = i < len(expected) - 1
|
||||
self.assertEqual(resp['IsTruncated'], expect_truncated)
|
||||
if expect_truncated:
|
||||
self.assertIsNotNone(resp['NextContinuationToken'])
|
||||
continuation_token = resp['NextContinuationToken']
|
||||
self._validate_object_listing(resp['Contents'],
|
||||
expected[i]['objects'],
|
||||
expect_owner=False)
|
||||
resp_subdirs = resp.get('CommonPrefixes', [])
|
||||
self.assertEqual(
|
||||
len(list(resp_objects)), len(expected[i]['objects']))
|
||||
for j, o in enumerate(resp_objects):
|
||||
self.assertEqual(o.find('Key').text,
|
||||
expected[i]['objects'][j].encode('utf-8'))
|
||||
self.assertTrue(o.find('LastModified').text is not None)
|
||||
self.assertRegexpMatches(
|
||||
o.find('LastModified').text,
|
||||
r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
|
||||
self.assertTrue(o.find('ETag').text is not None)
|
||||
self.assertTrue(o.find('Size').text is not None)
|
||||
self.assertEqual(o.find('StorageClass').text, 'STANDARD')
|
||||
self.assertIsNone(o.find('Owner/ID'))
|
||||
self.assertIsNone(o.find('Owner/DisplayName'))
|
||||
resp_subdirs = elem.findall('./CommonPrefixes')
|
||||
self.assertEqual(
|
||||
len(list(resp_subdirs)), len(expected[i]['subdirs']))
|
||||
for j, o in enumerate(resp_subdirs):
|
||||
self.assertEqual(
|
||||
o.find('Prefix').text,
|
||||
expected[i]['subdirs'][j].encode('utf-8'))
|
||||
resp_subdirs,
|
||||
[{'Prefix': p} for p in expected[i]['subdirs']])
|
||||
|
||||
def test_head_bucket_error(self):
|
||||
self.conn.make_request('PUT', 'bucket')
|
||||
event_system = self.conn.meta.events
|
||||
event_system.unregister(
|
||||
'before-parameter-build.s3',
|
||||
botocore.handlers.validate_bucket_name)
|
||||
|
||||
status, headers, body = \
|
||||
self.conn.make_request('HEAD', 'bucket+invalid')
|
||||
self.assertEqual(status, 400)
|
||||
self.assertEqual(body, '') # sanity
|
||||
self.conn.create_bucket(Bucket='bucket')
|
||||
|
||||
auth_error_conn = Connection(aws_secret_key='invalid')
|
||||
status, headers, body = \
|
||||
auth_error_conn.make_request('HEAD', 'bucket')
|
||||
self.assertEqual(status, 403)
|
||||
self.assertEqual(body, '') # sanity
|
||||
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
|
||||
self.conn.head_bucket(Bucket='bucket+invalid')
|
||||
self.assertEqual(
|
||||
ctx.exception.response['ResponseMetadata']['HTTPStatusCode'], 400)
|
||||
self.assertEqual(ctx.exception.response['Error']['Code'], '400')
|
||||
self.assertEqual(
|
||||
ctx.exception.response[
|
||||
'ResponseMetadata']['HTTPHeaders']['content-length'], '0')
|
||||
|
||||
status, headers, body = self.conn.make_request('HEAD', 'nothing')
|
||||
self.assertEqual(status, 404)
|
||||
self.assertEqual(body, '') # sanity
|
||||
auth_error_conn = get_boto3_conn(aws_secret_key='invalid')
|
||||
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
|
||||
auth_error_conn.head_bucket(Bucket='bucket')
|
||||
self.assertEqual(
|
||||
ctx.exception.response['ResponseMetadata']['HTTPStatusCode'], 403)
|
||||
self.assertEqual(
|
||||
ctx.exception.response['Error']['Code'], '403')
|
||||
self.assertEqual(
|
||||
ctx.exception.response[
|
||||
'ResponseMetadata']['HTTPHeaders']['content-length'], '0')
|
||||
|
||||
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
|
||||
self.conn.head_bucket(Bucket='nothing')
|
||||
self.assertEqual(
|
||||
ctx.exception.response['ResponseMetadata']['HTTPStatusCode'], 404)
|
||||
self.assertEqual(
|
||||
ctx.exception.response['Error']['Code'], '404')
|
||||
self.assertEqual(
|
||||
ctx.exception.response[
|
||||
'ResponseMetadata']['HTTPHeaders']['content-length'], '0')
|
||||
|
||||
def test_delete_bucket_error(self):
|
||||
status, headers, body = \
|
||||
self.conn.make_request('DELETE', 'bucket+invalid')
|
||||
self.assertEqual(get_error_code(body), 'InvalidBucketName')
|
||||
event_system = self.conn.meta.events
|
||||
event_system.unregister(
|
||||
'before-parameter-build.s3',
|
||||
botocore.handlers.validate_bucket_name)
|
||||
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
|
||||
self.conn.delete_bucket(Bucket='bucket+invalid')
|
||||
self.assertEqual(
|
||||
ctx.exception.response['Error']['Code'], 'InvalidBucketName')
|
||||
|
||||
auth_error_conn = Connection(aws_secret_key='invalid')
|
||||
status, headers, body = \
|
||||
auth_error_conn.make_request('DELETE', 'bucket')
|
||||
self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
|
||||
auth_error_conn = get_boto3_conn(aws_secret_key='invalid')
|
||||
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
|
||||
auth_error_conn.delete_bucket(Bucket='bucket')
|
||||
self.assertEqual(
|
||||
ctx.exception.response['Error']['Code'], 'SignatureDoesNotMatch')
|
||||
|
||||
status, headers, body = self.conn.make_request('DELETE', 'bucket')
|
||||
self.assertEqual(get_error_code(body), 'NoSuchBucket')
|
||||
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
|
||||
self.conn.delete_bucket(Bucket='bucket')
|
||||
self.assertEqual(
|
||||
ctx.exception.response['Error']['Code'], 'NoSuchBucket')
|
||||
|
||||
def test_bucket_invalid_method_error(self):
|
||||
def _mangle_req_method(request, **kwargs):
|
||||
request.method = 'GETPUT'
|
||||
|
||||
def _mangle_req_controller_method(request, **kwargs):
|
||||
request.method = '_delete_segments_bucket'
|
||||
|
||||
event_system = self.conn.meta.events
|
||||
event_system.register(
|
||||
'request-created.s3.CreateBucket',
|
||||
_mangle_req_method)
|
||||
# non existed verb in the controller
|
||||
status, headers, body = \
|
||||
self.conn.make_request('GETPUT', 'bucket')
|
||||
self.assertEqual(get_error_code(body), 'MethodNotAllowed')
|
||||
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
|
||||
self.conn.create_bucket(Bucket='bucket')
|
||||
self.assertEqual(
|
||||
ctx.exception.response['Error']['Code'], 'MethodNotAllowed')
|
||||
|
||||
event_system.unregister('request-created.s3.CreateBucket',
|
||||
_mangle_req_method)
|
||||
event_system.register('request-created.s3.CreateBucket',
|
||||
_mangle_req_controller_method)
|
||||
# the method exists in the controller but deny as MethodNotAllowed
|
||||
status, headers, body = \
|
||||
self.conn.make_request('_delete_segments_bucket', 'bucket')
|
||||
self.assertEqual(get_error_code(body), 'MethodNotAllowed')
|
||||
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
|
||||
self.conn.create_bucket(Bucket='bucket')
|
||||
self.assertEqual(
|
||||
ctx.exception.response['Error']['Code'], 'MethodNotAllowed')
|
||||
|
||||
|
||||
class TestS3ApiBucketSigV4(TestS3ApiBucket):
|
||||
|
||||
Loading…
Reference in New Issue
Block a user