openstack/swift
Code Issues Proposed changes

py3: mostly port s3 func tests

Browse Source 
test_bucket.py is proving somewhat problematic.

Change-Id: I5b337ef66a23fc989762801dd6a5ba1ed903f57b
This commit is contained in:
Tim Burke 2019-08-05 17:33:23 -07:00
parent 394d4655fa
commit f05119c16f
9 changed files with 105 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
.zuul.yaml
View File

@ -132,6 +132,18 @@
vars:
tox_envlist: func-domain-remap-staticweb-py3
- job:
name: swift-tox-func-s3api-py37
parent: swift-tox-func-py37
description: |
Run functional tests for swift under cPython version 3.7.
Uses tox with the ``func-s3api`` environment.
It sets TMPDIR to an XFS mount point created via
tools/test-setup.sh.
vars:
tox_envlist: func-s3api-py3
- job:
name: swift-tox-func-centos-7
parent: swift-tox-func
@ -480,6 +492,11 @@
- ^(api-ref|doc|releasenotes)/.*$
- ^test/probe/.*$
- ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$
- swift-tox-func-s3api-py37:
irrelevant-files:
- ^(api-ref|doc|releasenotes)/.*$
- ^test/probe/.*$
- ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$
# Other tests
- swift-tox-func-s3api-ceph-s3tests-tempauth:
@ -555,6 +572,7 @@
- swift-tox-func-encryption
- swift-tox-func-domain-remap-staticweb-py37
- swift-tox-func-ec-py37
- swift-tox-func-s3api-py37
- swift-probetests-centos-7:
irrelevant-files:
- ^(api-ref|releasenotes)/.*$

8
test/functional/s3api/s3_test_client.py
View File

@ -81,7 +81,7 @@ class Connection(object):
break
for bucket in buckets:
if not isinstance(bucket.name, six.binary_type):
if six.PY2 and not isinstance(bucket.name, bytes):
bucket.name = bucket.name.encode('utf-8')
try:
@ -103,7 +103,7 @@ class Connection(object):
exceptions.insert(0, 'Too many errors to continue:')
raise Exception('\n========\n'.join(exceptions))
def make_request(self, method, bucket='', obj='', headers=None, body='',
def make_request(self, method, bucket='', obj='', headers=None, body=b'',
query=None):
"""
Wrapper method of S3Connection.make_request.
@ -123,7 +123,9 @@ class Connection(object):
query_args=query, sender=None,
override_num_retries=RETRY_COUNT,
retry_handler=None)
return response.status, dict(response.getheaders()), response.read()
return (response.status,
{h.lower(): v for h, v in response.getheaders()},
response.read())
def generate_url_and_headers(self, method, bucket='', obj='',
expires_in=3600):

3
test/functional/s3api/test_acl.py
View File

@ -40,7 +40,8 @@ class TestS3Acl(S3ApiBase):
raise tf.SkipTest(
'TestS3Acl requires s3_access_key3 and s3_secret_key3 '
'configured for reduced-access user')
self.conn.make_request('PUT', self.bucket)
status, headers, body = self.conn.make_request('PUT', self.bucket)
self.assertEqual(status, 200, body)
access_key3 = tf.config['s3_access_key3']
secret_key3 = tf.config['s3_secret_key3']
self.conn3 = Connection(access_key3, secret_key3, access_key3)

61
test/functional/s3api/test_multi_upload.py
View File

@ -14,6 +14,7 @@
# limitations under the License.
import base64
import binascii
import unittest2
import os
import boto
@ -23,7 +24,7 @@ import boto
from distutils.version import StrictVersion
from hashlib import md5
from itertools import izip, izip_longest
from six.moves import zip, zip_longest
import test.functional as tf
from swift.common.middleware.s3api.etree import fromstring, tostring, Element, \
@ -67,7 +68,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
headers = [None] * len(keys)
self.conn.make_request('PUT', bucket)
query = 'uploads'
for key, key_headers in izip_longest(keys, headers):
for key, key_headers in zip_longest(keys, headers):
for i in range(trials):
status, resp_headers, body = \
self.conn.make_request('POST', bucket, key,
@ -76,7 +77,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
def _upload_part(self, bucket, key, upload_id, content=None, part_num=1):
query = 'partNumber=%s&uploadId=%s' % (part_num, upload_id)
content = content if content else 'a' * self.min_segment_size
content = content if content else b'a' * self.min_segment_size
status, headers, body = \
self.conn.make_request('PUT', bucket, key, body=content,
query=query)
@ -108,8 +109,9 @@ class TestS3ApiMultiUpload(S3ApiBase):
def test_object_multi_upload(self):
bucket = 'bucket'
keys = ['obj1', 'obj2', 'obj3']
bad_content_md5 = base64.b64encode(b'a' * 16).strip().decode('ascii')
headers = [None,
{'Content-MD5': base64.b64encode('a' * 16).strip()},
{'Content-MD5': bad_content_md5},
{'Etag': 'nonsense'}]
uploads = []
@ -118,20 +120,20 @@ class TestS3ApiMultiUpload(S3ApiBase):
# Initiate Multipart Upload
for expected_key, (status, headers, body) in \
izip(keys, results_generator):
self.assertEqual(status, 200)
zip(keys, results_generator):
self.assertEqual(status, 200, body)
self.assertCommonResponseHeaders(headers)
self.assertTrue('content-type' in headers)
self.assertIn('content-type', headers)
self.assertEqual(headers['content-type'], 'application/xml')
self.assertTrue('content-length' in headers)
self.assertIn('content-length', headers)
self.assertEqual(headers['content-length'], str(len(body)))
elem = fromstring(body, 'InitiateMultipartUploadResult')
self.assertEqual(elem.find('Bucket').text, bucket)
key = elem.find('Key').text
self.assertEqual(expected_key, key)
upload_id = elem.find('UploadId').text
self.assertTrue(upload_id is not None)
self.assertTrue((key, upload_id) not in uploads)
self.assertIsNotNone(upload_id)
self.assertNotIn((key, upload_id), uploads)
uploads.append((key, upload_id))
self.assertEqual(len(uploads), len(keys)) # sanity
@ -157,7 +159,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
self.assertEqual(elem.find('IsTruncated').text, 'false')
self.assertEqual(len(elem.findall('Upload')), 3)
for (expected_key, expected_upload_id), u in \
izip(uploads, elem.findall('Upload')):
zip(uploads, elem.findall('Upload')):
key = u.find('Key').text
upload_id = u.find('UploadId').text
self.assertEqual(expected_key, key)
@ -174,7 +176,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
# Upload Part
key, upload_id = uploads[0]
content = 'a' * self.min_segment_size
content = b'a' * self.min_segment_size
etag = md5(content).hexdigest()
status, headers, body = \
self._upload_part(bucket, key, upload_id, content)
@ -190,7 +192,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
key, upload_id = uploads[1]
src_bucket = 'bucket2'
src_obj = 'obj3'
src_content = 'b' * self.min_segment_size
src_content = b'b' * self.min_segment_size
etag = md5(src_content).hexdigest()
# prepare src obj
@ -266,7 +268,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
# etags will be used to generate xml for Complete Multipart Upload
etags = []
for (expected_etag, expected_date), p in \
izip(expected_parts_list, elem.findall('Part')):
zip(expected_parts_list, elem.findall('Part')):
last_modified = p.find('LastModified').text
self.assertTrue(last_modified is not None)
# TODO: sanity check
@ -295,9 +297,9 @@ class TestS3ApiMultiUpload(S3ApiBase):
else:
self.assertIn('transfer-encoding', headers)
self.assertEqual(headers['transfer-encoding'], 'chunked')
lines = body.split('\n')
self.assertTrue(lines[0].startswith('<?xml'), body)
self.assertTrue(lines[0].endswith('?>'), body)
lines = body.split(b'\n')
self.assertTrue(lines[0].startswith(b'<?xml'), body)
self.assertTrue(lines[0].endswith(b'?>'), body)
elem = fromstring(body, 'CompleteMultipartUploadResult')
# TODO: use tf.config value
self.assertEqual(
@ -305,9 +307,10 @@ class TestS3ApiMultiUpload(S3ApiBase):
elem.find('Location').text)
self.assertEqual(elem.find('Bucket').text, bucket)
self.assertEqual(elem.find('Key').text, key)
concatted_etags = ''.join(etag.strip('"') for etag in etags)
concatted_etags = b''.join(
etag.strip('"').encode('ascii') for etag in etags)
exp_etag = '"%s-%s"' % (
md5(concatted_etags.decode('hex')).hexdigest(), len(etags))
md5(binascii.unhexlify(concatted_etags)).hexdigest(), len(etags))
etag = elem.find('ETag').text
self.assertEqual(etag, exp_etag)
@ -332,7 +335,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
last_modified = elem.find('LastModified').text
self.assertIsNotNone(last_modified)
exp_content = 'a' * self.min_segment_size
exp_content = b'a' * self.min_segment_size
etag = md5(exp_content).hexdigest()
self.assertEqual(resp_etag, etag)
@ -723,7 +726,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
query = 'partNumber=%s&uploadId=%s' % (i, upload_id)
status, headers, body = \
self.conn.make_request('PUT', bucket, key, query=query,
body='A' * body_size[i])
body=b'A' * body_size[i])
etags.append(headers['etag'])
xml = self._gen_comp_xml(etags)
@ -747,7 +750,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
query = 'partNumber=%s&uploadId=%s' % (i, upload_id)
status, headers, body = \
self.conn.make_request('PUT', bucket, key, query=query,
body='A' * body_size[i])
body=b'A' * body_size[i])
etags.append(headers['etag'])
xml = self._gen_comp_xml(etags)
@ -770,9 +773,9 @@ class TestS3ApiMultiUpload(S3ApiBase):
etags = []
for i in range(1, 4):
query = 'partNumber=%s&uploadId=%s' % (2 * i - 1, upload_id)
status, headers, body = \
self.conn.make_request('PUT', bucket, key,
body='A' * 1024 * 1024 * 5, query=query)
status, headers, body = self.conn.make_request(
'PUT', bucket, key, body=b'A' * 1024 * 1024 * 5,
query=query)
etags.append(headers['etag'])
query = 'uploadId=%s' % upload_id
xml = self._gen_comp_xml(etags[:-1], step=2)
@ -791,7 +794,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
# Initiate Multipart Upload
for expected_key, (status, headers, body) in \
izip(keys, results_generator):
zip(keys, results_generator):
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers)
self.assertTrue('content-type' in headers)
@ -813,7 +816,7 @@ class TestS3ApiMultiUpload(S3ApiBase):
key, upload_id = uploads[0]
src_bucket = 'bucket2'
src_obj = 'obj4'
src_content = 'y' * (self.min_segment_size / 2) + 'z' * \
src_content = b'y' * (self.min_segment_size // 2) + b'z' * \
self.min_segment_size
src_range = 'bytes=0-%d' % (self.min_segment_size - 1)
etag = md5(src_content[:self.min_segment_size]).hexdigest()
@ -901,7 +904,7 @@ class TestS3ApiMultiUploadSigV4(TestS3ApiMultiUpload):
# Initiate Multipart Upload
for expected_key, (status, _, body) in \
izip(keys, results_generator):
zip(keys, results_generator):
self.assertEqual(status, 200) # sanity
elem = fromstring(body, 'InitiateMultipartUploadResult')
key = elem.find('Key').text
@ -915,7 +918,7 @@ class TestS3ApiMultiUploadSigV4(TestS3ApiMultiUpload):
# Upload Part
key, upload_id = uploads[0]
content = 'a' * self.min_segment_size
content = b'a' * self.min_segment_size
status, headers, body = \
self._upload_part(bucket, key, upload_id, content)
self.assertEqual(status, 200)

52
test/functional/s3api/test_object.py
View File

@ -25,7 +25,8 @@ import email.parser
from email.utils import formatdate, parsedate
from time import mktime
from hashlib import md5
from urllib import quote
import six
from six.moves.urllib.parse import quote
import test.functional as tf
@ -59,7 +60,7 @@ class TestS3ApiObject(S3ApiBase):
def test_object(self):
obj = 'object name with %-sign'
content = 'abc123'
content = b'abc123'
etag = md5(content).hexdigest()
# PUT Object
@ -219,19 +220,19 @@ class TestS3ApiObject(S3ApiBase):
status, headers, body = \
auth_error_conn.make_request('HEAD', self.bucket, obj)
self.assertEqual(status, 403)
self.assertEqual(body, '') # sanity
self.assertEqual(body, b'') # sanity
self.assertEqual(headers['content-type'], 'application/xml')
status, headers, body = \
self.conn.make_request('HEAD', self.bucket, 'invalid')
self.assertEqual(status, 404)
self.assertEqual(body, '') # sanity
self.assertEqual(body, b'') # sanity
self.assertEqual(headers['content-type'], 'application/xml')
status, headers, body = \
self.conn.make_request('HEAD', 'invalid', obj)
self.assertEqual(status, 404)
self.assertEqual(body, '') # sanity
self.assertEqual(body, b'') # sanity
self.assertEqual(headers['content-type'], 'application/xml')
def test_delete_object_error(self):
@ -265,7 +266,7 @@ class TestS3ApiObject(S3ApiBase):
def test_put_object_content_md5(self):
obj = 'object'
content = 'abcdefghij'
content = b'abcdefghij'
etag = md5(content).hexdigest()
headers = {'Content-MD5': calculate_md5(content)}
status, headers, body = \
@ -276,7 +277,7 @@ class TestS3ApiObject(S3ApiBase):
def test_put_object_content_type(self):
obj = 'object'
content = 'abcdefghij'
content = b'abcdefghij'
etag = md5(content).hexdigest()
headers = {'Content-Type': 'text/plain'}
status, headers, body = \
@ -290,7 +291,7 @@ class TestS3ApiObject(S3ApiBase):
def test_put_object_conditional_requests(self):
obj = 'object'
content = 'abcdefghij'
content = b'abcdefghij'
headers = {'If-None-Match': '*'}
status, headers, body = \
self.conn.make_request('PUT', self.bucket, obj, headers, content)
@ -318,7 +319,7 @@ class TestS3ApiObject(S3ApiBase):
def test_put_object_expect(self):
obj = 'object'
content = 'abcdefghij'
content = b'abcdefghij'
etag = md5(content).hexdigest()
headers = {'Expect': '100-continue'}
status, headers, body = \
@ -331,7 +332,7 @@ class TestS3ApiObject(S3ApiBase):
if expected_headers is None:
expected_headers = req_headers
obj = 'object'
content = 'abcdefghij'
content = b'abcdefghij'
etag = md5(content).hexdigest()
status, headers, body = \
self.conn.make_request('PUT', self.bucket, obj,
@ -387,7 +388,7 @@ class TestS3ApiObject(S3ApiBase):
def test_put_object_storage_class(self):
obj = 'object'
content = 'abcdefghij'
content = b'abcdefghij'
etag = md5(content).hexdigest()
headers = {'X-Amz-Storage-Class': 'STANDARD'}
status, headers, body = \
@ -399,7 +400,7 @@ class TestS3ApiObject(S3ApiBase):
def test_put_object_copy_source_params(self):
obj = 'object'
src_headers = {'X-Amz-Meta-Test': 'src'}
src_body = 'some content'
src_body = b'some content'
dst_bucket = 'dst-bucket'
dst_obj = 'dst_object'
self.conn.make_request('PUT', self.bucket, obj, src_headers, src_body)
@ -433,7 +434,7 @@ class TestS3ApiObject(S3ApiBase):
def test_put_object_copy_source(self):
obj = 'object'
content = 'abcdefghij'
content = b'abcdefghij'
etag = md5(content).hexdigest()
self.conn.make_request('PUT', self.bucket, obj, body=content)
@ -648,7 +649,7 @@ class TestS3ApiObject(S3ApiBase):
def test_get_object_range(self):
obj = 'object'
content = 'abcdefghij'
content = b'abcdefghij'
headers = {'x-amz-meta-test': 'swift'}
self.conn.make_request(
'PUT', self.bucket, obj, headers=headers, body=content)
@ -662,7 +663,7 @@ class TestS3ApiObject(S3ApiBase):
self.assertEqual(headers['content-length'], '5')
self.assertTrue('x-amz-meta-test' in headers)
self.assertEqual('swift', headers['x-amz-meta-test'])
self.assertEqual(body, 'bcdef')
self.assertEqual(body, b'bcdef')
headers = {'Range': 'bytes=5-'}
status, headers, body = \
@ -673,7 +674,7 @@ class TestS3ApiObject(S3ApiBase):
self.assertEqual(headers['content-length'], '5')
self.assertTrue('x-amz-meta-test' in headers)
self.assertEqual('swift', headers['x-amz-meta-test'])
self.assertEqual(body, 'fghij')
self.assertEqual(body, b'fghij')
headers = {'Range': 'bytes=-5'}
status, headers, body = \
@ -684,7 +685,7 @@ class TestS3ApiObject(S3ApiBase):
self.assertEqual(headers['content-length'], '5')
self.assertTrue('x-amz-meta-test' in headers)
self.assertEqual('swift', headers['x-amz-meta-test'])
self.assertEqual(body, 'fghij')
self.assertEqual(body, b'fghij')
ranges = ['1-2', '4-5']
@ -693,9 +694,9 @@ class TestS3ApiObject(S3ApiBase):
self.conn.make_request('GET', self.bucket, obj, headers=headers)
self.assertEqual(status, 206)
self.assertCommonResponseHeaders(headers)
self.assertTrue('content-length' in headers)
self.assertIn('content-length', headers)
self.assertTrue('content-type' in headers) # sanity
self.assertIn('content-type', headers) # sanity
content_type, boundary = headers['content-type'].split(';')
self.assertEqual('multipart/byteranges', content_type)
@ -704,10 +705,13 @@ class TestS3ApiObject(S3ApiBase):
# TODO: Using swift.common.utils.multipart_byteranges_to_document_iters
# could be easy enough.
parser = email.parser.FeedParser()
if six.PY2:
parser = email.parser.FeedParser()
else:
parser = email.parser.BytesFeedParser()
parser.feed(
"Content-Type: multipart/byterange; boundary=%s\r\n\r\n" %
boundary_str)
b"Content-Type: multipart/byterange; boundary=%s\r\n\r\n" %
boundary_str.encode('ascii'))
parser.feed(body)
message = parser.close()
@ -727,7 +731,7 @@ class TestS3ApiObject(S3ApiBase):
self.assertEqual(
expected_range, part.get('Content-Range'))
# rest
payload = part.get_payload().strip()
payload = part.get_payload(decode=True).strip()
self.assertEqual(content[start:end + 1], payload)
def test_get_object_if_modified_since(self):
@ -783,7 +787,7 @@ class TestS3ApiObject(S3ApiBase):
def test_head_object_range(self):
obj = 'object'
content = 'abcdefghij'
content = b'abcdefghij'
self.conn.make_request('PUT', self.bucket, obj, body=content)
headers = {'Range': 'bytes=1-5'}

8
test/functional/s3api/test_presigned.py
View File

@ -190,7 +190,7 @@ class TestS3ApiPresignedUrls(S3ApiBase):
# PUT empty object
put_url, headers = self.conn.generate_url_and_headers(
'PUT', bucket, obj)
resp = requests.put(put_url, data='', headers=headers)
resp = requests.put(put_url, data=b'', headers=headers)
self.assertEqual(resp.status_code, 200,
'Got %d %s' % (resp.status_code, resp.content))
# GET empty object
@ -199,10 +199,10 @@ class TestS3ApiPresignedUrls(S3ApiBase):
resp = requests.get(get_url, headers=headers)
self.assertEqual(resp.status_code, 200,
'Got %d %s' % (resp.status_code, resp.content))
self.assertEqual(resp.content, '')
self.assertEqual(resp.content, b'')
# PUT over object
resp = requests.put(put_url, data='foobar', headers=headers)
resp = requests.put(put_url, data=b'foobar', headers=headers)
self.assertEqual(resp.status_code, 200,
'Got %d %s' % (resp.status_code, resp.content))
@ -210,7 +210,7 @@ class TestS3ApiPresignedUrls(S3ApiBase):
resp = requests.get(get_url, headers=headers)
self.assertEqual(resp.status_code, 200,
'Got %d %s' % (resp.status_code, resp.content))
self.assertEqual(resp.content, 'foobar')
self.assertEqual(resp.content, b'foobar')
# DELETE Object
delete_url, headers = self.conn.generate_url_and_headers(

4
test/functional/s3api/test_service.py
View File

@ -80,8 +80,8 @@ class TestS3ApiService(S3ApiBase):
'GET', headers={'Date': '', 'x-amz-date': ''})
self.assertEqual(status, 403)
self.assertEqual(get_error_code(body), 'AccessDenied')
self.assertIn('AWS authentication requires a valid Date '
'or x-amz-date header', body)
self.assertIn(b'AWS authentication requires a valid Date '
b'or x-amz-date header', body)
class TestS3ApiServiceSigV4(TestS3ApiService):

3
test/functional/s3api/utils.py
View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from base64 import b64encode
from hashlib import md5
from swift.common.middleware.s3api.etree import fromstring
@ -28,4 +29,4 @@ def get_error_msg(body):
def calculate_md5(body):
return md5(body).digest().encode('base64').strip()
return b64encode(md5(body).digest()).strip().decode('ascii')

12
tox.ini
View File

@ -48,6 +48,12 @@ commands = ./.functests {posargs}
basepython = python3
commands =
nosetests {posargs: \
test/functional/s3api/test_acl.py \
test/functional/s3api/test_multi_delete.py \
test/functional/s3api/test_multi_upload.py \
test/functional/s3api/test_object.py \
test/functional/s3api/test_presigned.py \
test/functional/s3api/test_service.py \
test/functional/test_access_control.py \
test/functional/test_domain_remap.py \
test/functional/test_object.py \
@ -62,6 +68,12 @@ commands = {[testenv:func-py3]commands}
setenv = SWIFT_TEST_IN_PROCESS=1
SWIFT_TEST_IN_PROCESS_CONF_LOADER=ec
[testenv:func-s3api-py3]
basepython = python3
commands = {[testenv:func-py3]commands}
setenv = SWIFT_TEST_IN_PROCESS=1
SWIFT_TEST_IN_PROCESS_CONF_LOADER=s3api
[testenv:func-encryption-py3]
basepython = python3
commands = {[testenv:func-py3]commands}