diff --git a/swift/common/middleware/s3api/controllers/bucket.py b/swift/common/middleware/s3api/controllers/bucket.py
index b4f662acb5..350e7ba9c4 100644
--- a/swift/common/middleware/s3api/controllers/bucket.py
+++ b/swift/common/middleware/s3api/controllers/bucket.py
@@ -18,6 +18,7 @@ from base64 import standard_b64decode as b64decode
from six.moves.urllib.parse import quote
+from swift.common import swob
from swift.common.http import HTTP_OK
from swift.common.utils import json, public, config_true_value
@@ -66,8 +67,9 @@ class BucketController(Controller):
segments = json.loads(resp.body)
for seg in segments:
try:
- req.get_response(self.app, 'DELETE', container,
- seg['name'].encode('utf8'))
+ req.get_response(
+ self.app, 'DELETE', container,
+ swob.bytes_to_wsgi(seg['name'].encode('utf8')))
except NoSuchKey:
pass
except InternalError:
diff --git a/swift/common/middleware/s3api/controllers/multi_upload.py b/swift/common/middleware/s3api/controllers/multi_upload.py
index 91240a9159..10fd8ed68d 100644
--- a/swift/common/middleware/s3api/controllers/multi_upload.py
+++ b/swift/common/middleware/s3api/controllers/multi_upload.py
@@ -59,11 +59,14 @@ Static Large Object when the multipart upload is completed.
"""
+import binascii
from hashlib import md5
import os
import re
import time
+import six
+
from swift.common.swob import Range
from swift.common.utils import json, public, reiterate
from swift.common.db import utf8encode
@@ -222,8 +225,9 @@ class UploadsController(Controller):
:return (non_delimited_uploads, common_prefixes)
"""
- (prefix, delimiter) = \
- utf8encode(prefix, delimiter)
+ if six.PY2:
+ (prefix, delimiter) = \
+ utf8encode(prefix, delimiter)
non_delimited_uploads = []
common_prefixes = set()
for upload in uploads:
@@ -440,7 +444,7 @@ class UploadController(Controller):
# If the caller requested a list starting at a specific part number,
# construct a sub-set of the object list.
- objList = filter(filter_part_num_marker, objects)
+ objList = [obj for obj in objects if filter_part_num_marker(obj)]
# pylint: disable-msg=E1103
objList.sort(key=lambda o: int(o['name'].split('/')[-1]))
@@ -603,7 +607,7 @@ class UploadController(Controller):
'path': '/%s/%s/%s/%d' % (
container, req.object_name, upload_id, part_number),
'etag': etag})
- s3_etag_hasher.update(etag.decode('hex'))
+ s3_etag_hasher.update(binascii.a2b_hex(etag))
except (XMLSyntaxError, DocumentInvalid):
# NB: our schema definitions catch uploads with no parts here
raise MalformedXML()
@@ -661,8 +665,8 @@ class UploadController(Controller):
# ceph-s3tests happy
continue
if not yielded_anything:
- yield ('\n')
+ yield (b'\n')
yielded_anything = True
yield chunk
continue
@@ -708,8 +712,13 @@ class UploadController(Controller):
# in detail, https://github.com/boto/boto/pull/3513
parsed_url = urlparse(req.host_url)
host_url = '%s://%s' % (parsed_url.scheme, parsed_url.hostname)
- if parsed_url.port:
- host_url += ':%s' % parsed_url.port
+ # Why are we doing our own port parsing? Because py3 decided
+ # to start raising ValueErrors on access after parsing such
+ # an invalid port
+ netloc = parsed_url.netloc.split('@')[-1].split(']')[-1]
+ if ':' in netloc:
+ port = netloc.split(':', 2)[1]
+ host_url += ':%s' % port
SubElement(result_elem, 'Location').text = host_url + req.path
SubElement(result_elem, 'Bucket').text = req.container_name
@@ -717,13 +726,13 @@ class UploadController(Controller):
SubElement(result_elem, 'ETag').text = '"%s"' % s3_etag
resp.headers.pop('ETag', None)
if yielded_anything:
- yield '\n'
+ yield b'\n'
yield tostring(result_elem,
xml_declaration=not yielded_anything)
except ErrorResponse as err_resp:
if yielded_anything:
err_resp.xml_declaration = False
- yield '\n'
+ yield b'\n'
else:
# Oh good, we can still change HTTP status code, too!
resp.status = err_resp.status
diff --git a/swift/common/middleware/s3api/controllers/obj.py b/swift/common/middleware/s3api/controllers/obj.py
index 0a5a341d50..5a30c44dec 100644
--- a/swift/common/middleware/s3api/controllers/obj.py
+++ b/swift/common/middleware/s3api/controllers/obj.py
@@ -156,7 +156,7 @@ class ObjectController(Controller):
for chunk in resp.app_iter:
pass # drain the bulk-deleter response
resp.status = HTTP_NO_CONTENT
- resp.body = ''
+ resp.body = b''
except NoSuchKey:
# expect to raise NoSuchBucket when the bucket doesn't exist
req.get_container_info(self.app)
diff --git a/swift/common/middleware/s3api/etree.py b/swift/common/middleware/s3api/etree.py
index dcdd7f616d..29adbc38ed 100644
--- a/swift/common/middleware/s3api/etree.py
+++ b/swift/common/middleware/s3api/etree.py
@@ -120,7 +120,9 @@ class _Element(lxml.etree.ElementBase):
"""
utf-8 wrapper property of lxml.etree.Element.text
"""
- return utf8encode(lxml.etree.ElementBase.text.__get__(self))
+ if six.PY2:
+ return utf8encode(lxml.etree.ElementBase.text.__get__(self))
+ return lxml.etree.ElementBase.text.__get__(self)
@text.setter
def text(self, value):
diff --git a/swift/common/middleware/s3api/s3api.py b/swift/common/middleware/s3api/s3api.py
index c6bb27b04c..9682714f64 100644
--- a/swift/common/middleware/s3api/s3api.py
+++ b/swift/common/middleware/s3api/s3api.py
@@ -167,7 +167,7 @@ class ListingEtagMiddleware(object):
ctx._response_exc_info)
return [body]
- body = json.dumps(listing)
+ body = json.dumps(listing).encode('ascii')
ctx._response_headers[cl_index] = (
ctx._response_headers[cl_index][0],
str(len(body)),
@@ -237,7 +237,7 @@ class S3ApiMiddleware(object):
resp = err_resp
except Exception as e:
self.logger.exception(e)
- resp = InternalError(reason=e)
+ resp = InternalError(reason=str(e))
if isinstance(resp, S3ResponseBase) and 'swift.trans_id' in env:
resp.headers['x-amz-id-2'] = env['swift.trans_id']
diff --git a/swift/common/middleware/s3api/s3request.py b/swift/common/middleware/s3api/s3request.py
index 65051ebcdf..2ae7ac4f6a 100644
--- a/swift/common/middleware/s3api/s3request.py
+++ b/swift/common/middleware/s3api/s3request.py
@@ -14,6 +14,7 @@
# limitations under the License.
import base64
+import binascii
from collections import defaultdict, OrderedDict
from email.header import Header
from hashlib import sha1, sha256, md5
@@ -127,7 +128,8 @@ class HashingInput(object):
chunk = self._input.read(size)
self._hasher.update(chunk)
self._to_read -= len(chunk)
- if self._to_read < 0 or (size > len(chunk) and self._to_read) or (
+ short_read = bool(chunk) if size is None else (len(chunk) < size)
+ if self._to_read < 0 or (short_read and self._to_read) or (
self._to_read == 0 and
self._hasher.hexdigest() != self._expected):
self.close()
@@ -149,10 +151,10 @@ class SigV4Mixin(object):
def check_signature(self, secret):
secret = utf8encode(secret)
user_signature = self.signature
- derived_secret = 'AWS4' + secret
+ derived_secret = b'AWS4' + secret
for scope_piece in self.scope.values():
derived_secret = hmac.new(
- derived_secret, scope_piece, sha256).digest()
+ derived_secret, scope_piece.encode('utf8'), sha256).digest()
valid_signature = hmac.new(
derived_secret, self.string_to_sign, sha256).hexdigest()
return user_signature == valid_signature
@@ -331,10 +333,10 @@ class SigV4Mixin(object):
def _canonical_query_string(self):
return '&'.join(
- '%s=%s' % (quote(key, safe='-_.~'),
- quote(value, safe='-_.~'))
+ '%s=%s' % (swob.wsgi_quote(key, safe='-_.~'),
+ swob.wsgi_quote(value, safe='-_.~'))
for key, value in sorted(self.params.items())
- if key not in ('Signature', 'X-Amz-Signature'))
+ if key not in ('Signature', 'X-Amz-Signature')).encode('ascii')
def _headers_to_sign(self):
"""
@@ -383,7 +385,7 @@ class SigV4Mixin(object):
"""
It won't require bucket name in canonical_uri for v4.
"""
- return self.environ.get('RAW_PATH_INFO', self.path)
+ return swob.wsgi_to_bytes(self.environ.get('RAW_PATH_INFO', self.path))
def _canonical_request(self):
# prepare 'canonical_request'
@@ -401,7 +403,7 @@ class SigV4Mixin(object):
#
# 1. Add verb like: GET
- cr = [self.method.upper()]
+ cr = [swob.wsgi_to_bytes(self.method.upper())]
# 2. Add path like: /
path = self._canonical_uri()
@@ -415,12 +417,12 @@ class SigV4Mixin(object):
# host:iam.amazonaws.com
# x-amz-date:20150830T123600Z
headers_to_sign = self._headers_to_sign()
- cr.append(''.join('%s:%s\n' % (key, value)
- for key, value in headers_to_sign))
+ cr.append(b''.join(swob.wsgi_to_bytes('%s:%s\n' % (key, value))
+ for key, value in headers_to_sign))
# 5. Add signed headers into canonical request like
# content-type;host;x-amz-date
- cr.append(';'.join(k for k, v in headers_to_sign))
+ cr.append(b';'.join(swob.wsgi_to_bytes(k) for k, v in headers_to_sign))
# 6. Add payload string at the tail
if 'X-Amz-Credential' in self.params:
@@ -446,8 +448,8 @@ class SigV4Mixin(object):
# else, not provided -- Swift will kick out a 411 Length Required
# which will get translated back to a S3-style response in
# S3Request._swift_error_codes
- cr.append(hashed_payload)
- return '\n'.join(cr).encode('utf-8')
+ cr.append(swob.wsgi_to_bytes(hashed_payload))
+ return b'\n'.join(cr)
@property
def scope(self):
@@ -462,10 +464,11 @@ class SigV4Mixin(object):
"""
Create 'StringToSign' value in Amazon terminology for v4.
"""
- return '\n'.join(['AWS4-HMAC-SHA256',
- self.timestamp.amz_date_format,
- '/'.join(self.scope.values()),
- sha256(self._canonical_request()).hexdigest()])
+ return b'\n'.join([
+ b'AWS4-HMAC-SHA256',
+ self.timestamp.amz_date_format.encode('ascii'),
+ '/'.join(self.scope.values()).encode('utf8'),
+ sha256(self._canonical_request()).hexdigest().encode('ascii')])
def signature_does_not_match_kwargs(self):
kwargs = super(SigV4Mixin, self).signature_does_not_match_kwargs()
@@ -473,7 +476,7 @@ class SigV4Mixin(object):
kwargs.update({
'canonical_request': cr,
'canonical_request_bytes': ' '.join(
- format(ord(c), '02x') for c in cr),
+ format(ord(c), '02x') for c in cr.decode('latin1')),
})
return kwargs
@@ -545,6 +548,8 @@ class S3Request(swob.Request):
user_signature = self.signature
valid_signature = base64.b64encode(hmac.new(
secret, self.string_to_sign, sha1).digest()).strip()
+ if not six.PY2:
+ valid_signature = valid_signature.decode('ascii')
return user_signature == valid_signature
@property
@@ -613,7 +618,7 @@ class S3Request(swob.Request):
return None
def _parse_uri(self):
- if not check_utf8(self.environ['PATH_INFO']):
+ if not check_utf8(swob.wsgi_to_str(self.environ['PATH_INFO'])):
raise InvalidURI(self.path)
if self.bucket_in_host:
@@ -739,8 +744,10 @@ class S3Request(swob.Request):
# Non-base64-alphabet characters in value.
raise InvalidDigest(content_md5=value)
try:
- self.headers['ETag'] = value.decode('base64').encode('hex')
- except Exception:
+ self.headers['ETag'] = binascii.b2a_hex(
+ binascii.a2b_base64(value))
+ except binascii.error:
+ # incorrect padding, most likely
raise InvalidDigest(content_md5=value)
if len(self.headers['ETag']) != 32:
@@ -825,10 +832,11 @@ class S3Request(swob.Request):
'functionality that is not implemented',
header='Transfer-Encoding')
- if self.message_length() > max_length:
+ ml = self.message_length()
+ if ml and ml > max_length:
raise MalformedXML()
- if te or self.message_length():
+ if te or ml:
# Limit the read similar to how SLO handles manifests
body = self.body_file.read(max_length)
else:
@@ -843,7 +851,7 @@ class S3Request(swob.Request):
raise InvalidRequest('Missing required header for this request: '
'Content-MD5')
- digest = md5(body).digest().encode('base64').strip()
+ digest = base64.b64encode(md5(body).digest()).strip().decode('ascii')
if self.environ['HTTP_CONTENT_MD5'] != digest:
raise BadDigest(content_md5=self.environ['HTTP_CONTENT_MD5'])
@@ -927,9 +935,10 @@ class S3Request(swob.Request):
"""
amz_headers = {}
- buf = [self.method,
- _header_strip(self.headers.get('Content-MD5')) or '',
- _header_strip(self.headers.get('Content-Type')) or '']
+ buf = [swob.wsgi_to_bytes(wsgi_str) for wsgi_str in [
+ self.method,
+ _header_strip(self.headers.get('Content-MD5')) or '',
+ _header_strip(self.headers.get('Content-Type')) or '']]
if 'headers_raw' in self.environ: # eventlet >= 0.19.0
# See https://github.com/eventlet/eventlet/commit/67ec999
@@ -948,18 +957,18 @@ class S3Request(swob.Request):
if self._is_header_auth:
if 'x-amz-date' in amz_headers:
- buf.append('')
+ buf.append(b'')
elif 'Date' in self.headers:
- buf.append(self.headers['Date'])
+ buf.append(swob.wsgi_to_bytes(self.headers['Date']))
elif self._is_query_auth:
- buf.append(self.params['Expires'])
+ buf.append(swob.wsgi_to_bytes(self.params['Expires']))
else:
# Should have already raised NotS3Request in _parse_auth_info,
# but as a sanity check...
raise AccessDenied()
for key, value in sorted(amz_headers.items()):
- buf.append("%s:%s" % (key, value))
+ buf.append(swob.wsgi_to_bytes("%s:%s" % (key, value)))
path = self._canonical_uri()
if self.query_string:
@@ -971,10 +980,10 @@ class S3Request(swob.Request):
if key in ALLOWED_SUB_RESOURCES:
params.append('%s=%s' % (key, value) if value else key)
if params:
- buf.append('%s?%s' % (path, '&'.join(params)))
+ buf.append(swob.wsgi_to_bytes('%s?%s' % (path, '&'.join(params))))
else:
- buf.append(path)
- return '\n'.join(buf)
+ buf.append(swob.wsgi_to_bytes(path))
+ return b'\n'.join(buf)
def signature_does_not_match_kwargs(self):
return {
@@ -982,7 +991,8 @@ class S3Request(swob.Request):
'string_to_sign': self.string_to_sign,
'signature_provided': self.signature,
'string_to_sign_bytes': ' '.join(
- format(ord(c), '02x') for c in self.string_to_sign),
+ format(ord(c), '02x')
+ for c in self.string_to_sign.decode('latin1')),
}
@property
@@ -1320,7 +1330,6 @@ class S3Request(swob.Request):
# reuse account and tokens
_, self.account, _ = split_path(sw_resp.environ['PATH_INFO'],
2, 3, True)
- self.account = utf8encode(self.account)
resp = S3Response.from_swift_resp(sw_resp)
status = resp.status_int # pylint: disable-msg=E1101
@@ -1354,7 +1363,7 @@ class S3Request(swob.Request):
raise err_resp()
if status == HTTP_BAD_REQUEST:
- raise BadSwiftRequest(err_msg)
+ raise BadSwiftRequest(err_msg.decode('utf8'))
if status == HTTP_UNAUTHORIZED:
raise SignatureDoesNotMatch(
**self.signature_does_not_match_kwargs())
@@ -1487,7 +1496,6 @@ class S3AclRequest(S3Request):
_, self.account, _ = split_path(sw_resp.environ['PATH_INFO'],
2, 3, True)
- self.account = utf8encode(self.account)
if 'HTTP_X_USER_NAME' in sw_resp.environ:
# keystone
diff --git a/swift/common/middleware/s3api/s3response.py b/swift/common/middleware/s3api/s3response.py
index 9e6a759bd7..b71efa7f18 100644
--- a/swift/common/middleware/s3api/s3response.py
+++ b/swift/common/middleware/s3api/s3response.py
@@ -243,8 +243,10 @@ class ErrorResponse(S3ResponseBase, swob.HTTPException):
if isinstance(value, (dict, MutableMapping)):
self._dict_to_etree(elem, value)
else:
+ if isinstance(value, (int, float, bool)):
+ value = str(value)
try:
- elem.text = str(value)
+ elem.text = value
except ValueError:
# We set an invalid string for XML.
elem.text = '(invalid string)'
diff --git a/swift/common/middleware/s3api/subresource.py b/swift/common/middleware/s3api/subresource.py
index 42bd67f003..c85d0b7b33 100644
--- a/swift/common/middleware/s3api/subresource.py
+++ b/swift/common/middleware/s3api/subresource.py
@@ -43,6 +43,8 @@ http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html
"""
from functools import partial
+import six
+
from swift.common.utils import json
from swift.common.middleware.s3api.s3response import InvalidArgument, \
@@ -218,6 +220,11 @@ class User(Grantee):
def __str__(self):
return self.display_name
+ def __lt__(self, other):
+ if not isinstance(other, User):
+ return NotImplemented
+ return self.id < other.id
+
class Owner(object):
"""
@@ -415,9 +422,14 @@ class ACL(object):
self.s3_acl = s3_acl
self.allow_no_owner = allow_no_owner
- def __repr__(self):
+ def __bytes__(self):
return tostring(self.elem())
+ def __repr__(self):
+ if six.PY2:
+ return self.__bytes__()
+ return self.__bytes__().decode('utf8')
+
@classmethod
def from_elem(cls, elem, s3_acl=False, allow_no_owner=False):
"""
diff --git a/swift/common/swob.py b/swift/common/swob.py
index 9a157241df..4415b12478 100644
--- a/swift/common/swob.py
+++ b/swift/common/swob.py
@@ -294,15 +294,15 @@ def str_to_wsgi(native_str):
return bytes_to_wsgi(native_str.encode('utf8', errors='surrogateescape'))
-def wsgi_quote(wsgi_str):
+def wsgi_quote(wsgi_str, safe='/'):
if six.PY2:
if not isinstance(wsgi_str, bytes):
raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
- return urllib.parse.quote(wsgi_str)
+ return urllib.parse.quote(wsgi_str, safe=safe)
if not isinstance(wsgi_str, str) or any(ord(x) > 255 for x in wsgi_str):
raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
- return urllib.parse.quote(wsgi_str, encoding='latin-1')
+ return urllib.parse.quote(wsgi_str, safe=safe, encoding='latin-1')
def wsgi_unquote(wsgi_str):
@@ -463,6 +463,10 @@ def _resp_app_iter_property():
def setter(self, value):
if isinstance(value, (list, tuple)):
+ for i, item in enumerate(value):
+ if not isinstance(item, bytes):
+ raise TypeError('WSGI responses must be bytes; '
+ 'got %s for item %d' % (type(item), i))
self.content_length = sum(map(len, value))
elif value is not None:
self.content_length = None
diff --git a/test/unit/common/middleware/s3api/helpers.py b/test/unit/common/middleware/s3api/helpers.py
index 54051889a0..525dc35111 100644
--- a/test/unit/common/middleware/s3api/helpers.py
+++ b/test/unit/common/middleware/s3api/helpers.py
@@ -165,12 +165,16 @@ class FakeSwift(object):
# keep old sysmeta for s3acl
headers.update({key: value})
+ if body is not None and not isinstance(body, (bytes, list)):
+ body = body.encode('utf8')
self._responses[(method, path)] = (response_class, headers, body)
def register_unconditionally(self, method, path, response_class, headers,
body):
# register() keeps old sysmeta around, but
# register_unconditionally() keeps nothing.
+ if body is not None and not isinstance(body, bytes):
+ body = body.encode('utf8')
self._responses[(method, path)] = (response_class, headers, body)
def clear_calls(self):
diff --git a/test/unit/common/middleware/s3api/test_acl.py b/test/unit/common/middleware/s3api/test_acl.py
index f7e800e5da..9037b32c88 100644
--- a/test/unit/common/middleware/s3api/test_acl.py
+++ b/test/unit/common/middleware/s3api/test_acl.py
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import base64
import unittest
import mock
@@ -131,8 +132,8 @@ class TestS3ApiAcl(S3ApiTestCase):
'UnexpectedContent')
def _test_put_no_body(self, use_content_length=False,
- use_transfer_encoding=False, string_to_md5=''):
- content_md5 = md5(string_to_md5).digest().encode('base64').strip()
+ use_transfer_encoding=False, string_to_md5=b''):
+ content_md5 = base64.b64encode(md5(string_to_md5).digest()).strip()
with UnreadableInput(self) as fake_input:
req = Request.blank(
'/bucket?acl',
@@ -153,16 +154,17 @@ class TestS3ApiAcl(S3ApiTestCase):
self.assertEqual(self._get_error_code(body), 'MissingSecurityHeader')
self.assertEqual(self._get_error_message(body),
'Your request was missing a required header.')
- self.assertIn('x-amz-acl', body)
+ self.assertIn(b'x-amz-acl',
+ body)
@s3acl
def test_bucket_fails_with_neither_acl_header_nor_xml_PUT(self):
self._test_put_no_body()
- self._test_put_no_body(string_to_md5='test')
+ self._test_put_no_body(string_to_md5=b'test')
self._test_put_no_body(use_content_length=True)
- self._test_put_no_body(use_content_length=True, string_to_md5='test')
+ self._test_put_no_body(use_content_length=True, string_to_md5=b'test')
self._test_put_no_body(use_transfer_encoding=True)
- self._test_put_no_body(use_transfer_encoding=True, string_to_md5='zz')
+ self._test_put_no_body(use_transfer_encoding=True, string_to_md5=b'zz')
def test_object_acl_GET(self):
req = Request.blank('/bucket/object?acl',
diff --git a/test/unit/common/middleware/s3api/test_bucket.py b/test/unit/common/middleware/s3api/test_bucket.py
index 85dd7437cd..d530596e88 100644
--- a/test/unit/common/middleware/s3api/test_bucket.py
+++ b/test/unit/common/middleware/s3api/test_bucket.py
@@ -17,6 +17,7 @@ import unittest
import cgi
import mock
+import six
from six.moves.urllib.parse import quote
from swift.common import swob
@@ -62,7 +63,8 @@ class TestS3ApiBucket(S3ApiTestCase):
for name, _, _, _ in self.objects:
self.swift.register(
'DELETE',
- '/v1/AUTH_test/bucket+segments/' + name.encode('utf-8'),
+ '/v1/AUTH_test/bucket+segments/' +
+ swob.bytes_to_wsgi(name.encode('utf-8')),
swob.HTTPNoContent, {}, json.dumps([]))
self.swift.register(
'GET',
@@ -118,7 +120,7 @@ class TestS3ApiBucket(S3ApiTestCase):
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '404')
- self.assertEqual(body, '') # sanity
+ self.assertEqual(body, b'') # sanity
def test_bucket_HEAD_slash(self):
req = Request.blank('/junk/',
@@ -168,7 +170,8 @@ class TestS3ApiBucket(S3ApiTestCase):
self.assertEqual('2011-01-05T02:19:14.275Z',
o.find('./LastModified').text)
self.assertEqual(items, [
- (i[0].encode('utf-8'), '"0-N"' if i[0] == 'slo' else '"0"')
+ (i[0].encode('utf-8') if six.PY2 else i[0],
+ '"0-N"' if i[0] == 'slo' else '"0"')
for i in self.objects])
def test_bucket_GET_url_encoded(self):
@@ -519,8 +522,12 @@ class TestS3ApiBucket(S3ApiTestCase):
self.assertEqual(elem.findall('./DeleteMarker'), [])
versions = elem.findall('./Version')
objects = list(self.objects)
- self.assertEqual([v.find('./Key').text for v in versions],
- [v[0].encode('utf-8') for v in objects])
+ if six.PY2:
+ self.assertEqual([v.find('./Key').text for v in versions],
+ [v[0].encode('utf-8') for v in objects])
+ else:
+ self.assertEqual([v.find('./Key').text for v in versions],
+ [v[0] for v in objects])
self.assertEqual([v.find('./IsLatest').text for v in versions],
['true' for v in objects])
self.assertEqual([v.find('./VersionId').text for v in versions],
@@ -598,7 +605,7 @@ class TestS3ApiBucket(S3ApiTestCase):
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
- self.assertEqual(body, '')
+ self.assertEqual(body, b'')
self.assertEqual(status.split()[0], '200')
self.assertEqual(headers['Location'], '/bucket')
@@ -610,7 +617,7 @@ class TestS3ApiBucket(S3ApiTestCase):
'Date': self.get_date_header(),
'Transfer-Encoding': 'chunked'})
status, headers, body = self.call_s3api(req)
- self.assertEqual(body, '')
+ self.assertEqual(body, b'')
self.assertEqual(status.split()[0], '200')
self.assertEqual(headers['Location'], '/bucket')
@@ -622,7 +629,7 @@ class TestS3ApiBucket(S3ApiTestCase):
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
- self.assertEqual(body, '')
+ self.assertEqual(body, b'')
self.assertEqual(status.split()[0], '200')
self.assertEqual(headers['Location'], '/bucket')
diff --git a/test/unit/common/middleware/s3api/test_etree.py b/test/unit/common/middleware/s3api/test_etree.py
index be2249ae02..63cb5c7ffc 100644
--- a/test/unit/common/middleware/s3api/test_etree.py
+++ b/test/unit/common/middleware/s3api/test_etree.py
@@ -15,6 +15,8 @@
import unittest
+import six
+
from swift.common.middleware.s3api import etree
@@ -58,15 +60,18 @@ class TestS3ApiEtree(unittest.TestCase):
sub.text = '\xef\xbc\xa1'
self.assertTrue(isinstance(sub.text, str))
xml_string = etree.tostring(elem)
- self.assertTrue(isinstance(xml_string, str))
+ self.assertIsInstance(xml_string, bytes)
def test_fromstring_with_nonascii_text(self):
- input_str = '\n' \
- '\xef\xbc\xa1'
+ input_str = b'\n' \
+ b'\xef\xbc\xa1'
elem = etree.fromstring(input_str)
text = elem.find('FOO').text
- self.assertEqual(text, '\xef\xbc\xa1')
- self.assertTrue(isinstance(text, str))
+ if six.PY2:
+ self.assertEqual(text, b'\xef\xbc\xa1')
+ else:
+ self.assertEqual(text, b'\xef\xbc\xa1'.decode('utf8'))
+ self.assertIsInstance(text, str)
if __name__ == '__main__':
diff --git a/test/unit/common/middleware/s3api/test_multi_delete.py b/test/unit/common/middleware/s3api/test_multi_delete.py
index c8bb7206f6..6ea5f59003 100644
--- a/test/unit/common/middleware/s3api/test_multi_delete.py
+++ b/test/unit/common/middleware/s3api/test_multi_delete.py
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import base64
import json
import unittest
from datetime import datetime
@@ -43,7 +44,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = 'object'
body = tostring(elem, use_s3ns=False)
- content_md5 = md5(body).digest().encode('base64').strip()
+ content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket/object?delete',
environ={'REQUEST_METHOD': 'POST'},
@@ -80,7 +81,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key
body = tostring(elem, use_s3ns=False)
- content_md5 = md5(body).digest().encode('base64').strip()
+ content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
@@ -133,7 +134,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key
body = tostring(elem, use_s3ns=False)
- content_md5 = md5(body).digest().encode('base64').strip()
+ content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
@@ -180,7 +181,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key
body = tostring(elem, use_s3ns=False)
- content_md5 = md5(body).digest().encode('base64').strip()
+ content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
@@ -207,7 +208,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key')
body = tostring(elem, use_s3ns=False)
- content_md5 = md5(body).digest().encode('base64').strip()
+ content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
@@ -232,7 +233,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
SubElement(obj, 'Key').text = key
SubElement(obj, 'VersionId').text = 'not-supported'
body = tostring(elem, use_s3ns=False)
- content_md5 = md5(body).digest().encode('base64').strip()
+ content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
@@ -286,7 +287,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = name
body = tostring(elem, use_s3ns=False)
- content_md5 = md5(body).digest().encode('base64').strip()
+ content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
@@ -308,7 +309,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = 'x' * 1000 + str(i)
body = tostring(elem, use_s3ns=False)
- content_md5 = md5(body).digest().encode('base64').strip()
+ content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
@@ -333,7 +334,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key
body = tostring(elem, use_s3ns=False)
- content_md5 = md5(body).digest().encode('base64').strip()
+ content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
@@ -374,8 +375,8 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
self.assertEqual(len(elem.findall('Deleted')), len(self.keys))
def _test_no_body(self, use_content_length=False,
- use_transfer_encoding=False, string_to_md5=''):
- content_md5 = md5(string_to_md5).digest().encode('base64').strip()
+ use_transfer_encoding=False, string_to_md5=b''):
+ content_md5 = base64.b64encode(md5(string_to_md5).digest()).strip()
with UnreadableInput(self) as fake_input:
req = Request.blank(
'/bucket?delete',
@@ -398,11 +399,11 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
@s3acl
def test_object_multi_DELETE_empty_body(self):
self._test_no_body()
- self._test_no_body(string_to_md5='test')
+ self._test_no_body(string_to_md5=b'test')
self._test_no_body(use_content_length=True)
- self._test_no_body(use_content_length=True, string_to_md5='test')
+ self._test_no_body(use_content_length=True, string_to_md5=b'test')
self._test_no_body(use_transfer_encoding=True)
- self._test_no_body(use_transfer_encoding=True, string_to_md5='test')
+ self._test_no_body(use_transfer_encoding=True, string_to_md5=b'test')
if __name__ == '__main__':
unittest.main()
diff --git a/test/unit/common/middleware/s3api/test_multi_upload.py b/test/unit/common/middleware/s3api/test_multi_upload.py
index 2dc3d8db61..cff18c2759 100644
--- a/test/unit/common/middleware/s3api/test_multi_upload.py
+++ b/test/unit/common/middleware/s3api/test_multi_upload.py
@@ -14,12 +14,13 @@
# limitations under the License.
import base64
+import binascii
import hashlib
from mock import patch
import os
import time
import unittest
-from urllib import quote
+from six.moves.urllib.parse import quote
from swift.common import swob
from swift.common.swob import Request
@@ -66,9 +67,9 @@ MULTIPARTS_TEMPLATE = \
('subdir/object/Z/2', '2014-05-07T19:47:58.592270', 'fedcba9876543210',
41))
-S3_ETAG = '"%s-2"' % hashlib.md5((
+S3_ETAG = '"%s-2"' % hashlib.md5(binascii.a2b_hex(
'0123456789abcdef0123456789abcdef'
- 'fedcba9876543210fedcba9876543210').decode('hex')).hexdigest()
+ 'fedcba9876543210fedcba9876543210')).hexdigest()
class TestS3ApiMultiUpload(S3ApiTestCase):
@@ -83,9 +84,9 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
self.s3api.conf.min_segment_size = 1
- objects = map(lambda item: {'name': item[0], 'last_modified': item[1],
- 'hash': item[2], 'bytes': item[3]},
- OBJECTS_TEMPLATE)
+ objects = [{'name': item[0], 'last_modified': item[1],
+ 'hash': item[2], 'bytes': item[3]}
+ for item in OBJECTS_TEMPLATE]
object_list = json.dumps(objects)
self.swift.register('PUT', segment_bucket,
@@ -172,10 +173,10 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
multiparts=None):
segment_bucket = '/v1/AUTH_test/bucket+segments'
objects = multiparts or MULTIPARTS_TEMPLATE
- objects = map(lambda item: {'name': item[0], 'last_modified': item[1],
- 'hash': item[2], 'bytes': item[3]},
- objects)
- object_list = json.dumps(objects)
+ objects = [{'name': item[0], 'last_modified': item[1],
+ 'hash': item[2], 'bytes': item[3]}
+ for item in objects]
+ object_list = json.dumps(objects).encode('ascii')
self.swift.register('GET', segment_bucket, swob.HTTPOk, {},
object_list)
@@ -568,7 +569,7 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
self._test_object_multipart_upload_initiate({})
self._test_object_multipart_upload_initiate({'Etag': 'blahblahblah'})
self._test_object_multipart_upload_initiate({
- 'Content-MD5': base64.b64encode('blahblahblahblah').strip()})
+ 'Content-MD5': base64.b64encode(b'blahblahblahblah').strip()})
@s3acl(s3acl_only=True)
@patch('swift.common.middleware.s3api.controllers.multi_upload.'
@@ -667,7 +668,8 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
self.assertEqual(self._get_error_code(body), 'NoSuchBucket')
def test_object_multipart_upload_complete(self):
- content_md5 = base64.b64encode(hashlib.md5(XML).digest())
+ content_md5 = base64.b64encode(hashlib.md5(
+ XML.encode('ascii')).digest())
req = Request.blank('/bucket/object?uploadId=X',
environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization': 'AWS test:tester:hmac',
@@ -701,7 +703,8 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
self.assertEqual(headers.get(h), override_etag)
def test_object_multipart_upload_invalid_md5(self):
- bad_md5 = base64.b64encode(hashlib.md5(XML + 'some junk').digest())
+ bad_md5 = base64.b64encode(hashlib.md5(
+ XML.encode('ascii') + b'some junk').digest())
req = Request.blank('/bucket/object?uploadId=X',
environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization': 'AWS test:tester:hmac',
@@ -726,11 +729,11 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
]))
self.swift.register(
'PUT', '/v1/AUTH_test/bucket/heartbeat-ok',
- swob.HTTPAccepted, {}, [' ', ' ', ' ', json.dumps({
+ swob.HTTPAccepted, {}, [b' ', b' ', b' ', json.dumps({
'Etag': '"slo-etag"',
'Response Status': '201 Created',
'Errors': [],
- })])
+ }).encode('ascii')])
mock_time.time.side_effect = (
1, # start_time
12, # first whitespace
@@ -748,14 +751,14 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
'Date': self.get_date_header(), },
body=XML)
status, headers, body = self.call_s3api(req)
- lines = body.split('\n')
- self.assertTrue(lines[0].startswith('%s' % S3_ETAG, body)
+ self.assertIn(('%s' % S3_ETAG).encode('ascii'), body)
self.assertEqual(self.swift.calls, [
('HEAD', '/v1/AUTH_test/bucket'),
('HEAD', '/v1/AUTH_test/bucket+segments/heartbeat-ok/X'),
@@ -779,10 +782,10 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
]))
self.swift.register(
'PUT', '/v1/AUTH_test/bucket/heartbeat-fail',
- swob.HTTPAccepted, {}, [' ', ' ', ' ', json.dumps({
+ swob.HTTPAccepted, {}, [b' ', b' ', b' ', json.dumps({
'Response Status': '400 Bad Request',
'Errors': [['some/object', '403 Forbidden']],
- })])
+ }).encode('ascii')])
mock_time.time.side_effect = (
1, # start_time
12, # first whitespace
@@ -797,8 +800,8 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
'Date': self.get_date_header(), },
body=XML)
status, headers, body = self.call_s3api(req)
- lines = body.split('\n')
- self.assertTrue(lines[0].startswith('us-east-1')
+ b'us-east-1')
test('test:tester/%s/us-east-1/not-s3/aws4_request' % dt,
'Error parsing the X-Amz-Credential parameter; incorrect service '
'"not-s3". This endpoint belongs to "s3".')
@@ -483,9 +485,9 @@ class TestS3ApiMiddleware(S3ApiTestCase):
self.assertEqual(req.environ['s3api.auth_details'], {
'access_key': 'test:tester',
'signature': 'hmac',
- 'string_to_sign': '\n'.join([
- 'PUT', '', '', date_header,
- '/bucket/object?partNumber=1&uploadId=123456789abcdef']),
+ 'string_to_sign': b'\n'.join([
+ b'PUT', b'', b'', date_header.encode('ascii'),
+ b'/bucket/object?partNumber=1&uploadId=123456789abcdef']),
'check_signature': mock_cs})
def test_invalid_uri(self):
@@ -506,8 +508,10 @@ class TestS3ApiMiddleware(S3ApiTestCase):
self.assertEqual(self._get_error_code(body), 'InvalidDigest')
def test_object_create_bad_md5_too_short(self):
- too_short_digest = hashlib.md5('hey').hexdigest()[:-1]
- md5_str = too_short_digest.encode('base64').strip()
+ too_short_digest = hashlib.md5(b'hey').digest()[:-1]
+ md5_str = base64.b64encode(too_short_digest).strip()
+ if not six.PY2:
+ md5_str = md5_str.decode('ascii')
req = Request.blank(
'/bucket/object',
environ={'REQUEST_METHOD': 'PUT',
@@ -518,8 +522,10 @@ class TestS3ApiMiddleware(S3ApiTestCase):
self.assertEqual(self._get_error_code(body), 'InvalidDigest')
def test_object_create_bad_md5_too_long(self):
- too_long_digest = hashlib.md5('hey').hexdigest() + 'suffix'
- md5_str = too_long_digest.encode('base64').strip()
+ too_long_digest = hashlib.md5(b'hey').digest() + b'suffix'
+ md5_str = base64.b64encode(too_long_digest).strip()
+ if not six.PY2:
+ md5_str = md5_str.decode('ascii')
req = Request.blank(
'/bucket/object',
environ={'REQUEST_METHOD': 'PUT',
@@ -705,13 +711,13 @@ class TestS3ApiMiddleware(S3ApiTestCase):
with self.assertRaises(ValueError) as cm:
self.s3api.check_pipeline(self.conf)
self.assertIn('expected auth between s3api and proxy-server',
- cm.exception.message)
+ cm.exception.args[0])
pipeline.return_value = 'proxy-server'
with self.assertRaises(ValueError) as cm:
self.s3api.check_pipeline(self.conf)
self.assertIn("missing filters ['s3api']",
- cm.exception.message)
+ cm.exception.args[0])
def test_s3api_initialization_with_disabled_pipeline_check(self):
with patch("swift.common.middleware.s3api.s3api.loadcontext"), \
@@ -799,7 +805,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
'Missing required header for this request: x-amz-content-sha256')
def test_signature_v4_bad_authorization_string(self):
- def test(auth_str, error, msg, extra=''):
+ def test(auth_str, error, msg, extra=b''):
environ = {
'REQUEST_METHOD': 'GET'}
headers = {
@@ -835,7 +841,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
test(auth_str, 'AuthorizationHeaderMalformed',
"The authorization header is malformed; "
"the region 'us-west-2' is wrong; expecting 'us-east-1'",
- 'us-east-1')
+ b'us-east-1')
auth_str = ('AWS4-HMAC-SHA256 '
'Credential=test:tester/%s/us-east-1/not-s3/aws4_request, '
@@ -901,8 +907,8 @@ class TestS3ApiMiddleware(S3ApiTestCase):
patch.object(swift.common.middleware.s3api.s3request,
'SERVICE', 'host'):
req = _get_req(path, environ)
- hash_in_sts = req._string_to_sign().split('\n')[3]
- self.assertEqual(hash_val, hash_in_sts)
+ hash_in_sts = req._string_to_sign().split(b'\n')[3]
+ self.assertEqual(hash_val, hash_in_sts.decode('ascii'))
self.assertTrue(req.check_signature(
'wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY'))
@@ -963,7 +969,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
'validate_bucket_name'):
verify('27ba31df5dbc6e063d8f87d62eb07143'
'f7f271c5330a917840586ac1c85b6f6b',
- unquote('/%E1%88%B4'), env)
+ swob.wsgi_unquote('/%E1%88%B4'), env)
# get-vanilla-query-order-key
env = {
@@ -1101,12 +1107,12 @@ class TestS3ApiMiddleware(S3ApiTestCase):
swob.HTTPOk, {}, None)
with patch.object(self.s3_token, '_json_request') as mock_req:
mock_resp = requests.Response()
- mock_resp._content = json.dumps(GOOD_RESPONSE_V2)
+ mock_resp._content = json.dumps(GOOD_RESPONSE_V2).encode('ascii')
mock_resp.status_code = 201
mock_req.return_value = mock_resp
status, headers, body = self.call_s3api(req)
- self.assertEqual(body, '')
+ self.assertEqual(body, b'')
self.assertEqual(1, mock_req.call_count)
def test_s3api_with_only_s3_token_v3(self):
@@ -1127,12 +1133,12 @@ class TestS3ApiMiddleware(S3ApiTestCase):
swob.HTTPOk, {}, None)
with patch.object(self.s3_token, '_json_request') as mock_req:
mock_resp = requests.Response()
- mock_resp._content = json.dumps(GOOD_RESPONSE_V3)
+ mock_resp._content = json.dumps(GOOD_RESPONSE_V3).encode('ascii')
mock_resp.status_code = 200
mock_req.return_value = mock_resp
status, headers, body = self.call_s3api(req)
- self.assertEqual(body, '')
+ self.assertEqual(body, b'')
self.assertEqual(1, mock_req.call_count)
def test_s3api_with_s3_token_and_auth_token(self):
@@ -1157,7 +1163,8 @@ class TestS3ApiMiddleware(S3ApiTestCase):
with patch.object(self.auth_token,
'_do_fetch_token') as mock_fetch:
mock_resp = requests.Response()
- mock_resp._content = json.dumps(GOOD_RESPONSE_V2)
+ mock_resp._content = json.dumps(
+ GOOD_RESPONSE_V2).encode('ascii')
mock_resp.status_code = 201
mock_req.return_value = mock_resp
@@ -1167,7 +1174,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
mock_fetch.return_value = (MagicMock(), mock_access_info)
status, headers, body = self.call_s3api(req)
- self.assertEqual(body, '')
+ self.assertEqual(body, b'')
self.assertEqual(1, mock_req.call_count)
# With X-Auth-Token, auth_token will call _do_fetch_token to
# connect to keystone in auth_token, again
@@ -1198,7 +1205,8 @@ class TestS3ApiMiddleware(S3ApiTestCase):
no_token_id_good_resp = copy.deepcopy(GOOD_RESPONSE_V2)
# delete token id
del no_token_id_good_resp['access']['token']['id']
- mock_resp._content = json.dumps(no_token_id_good_resp)
+ mock_resp._content = json.dumps(
+ no_token_id_good_resp).encode('ascii')
mock_resp.status_code = 201
mock_req.return_value = mock_resp
diff --git a/test/unit/common/middleware/s3api/test_s3request.py b/test/unit/common/middleware/s3api/test_s3request.py
index 3cec613586..f9e4acbf53 100644
--- a/test/unit/common/middleware/s3api/test_s3request.py
+++ b/test/unit/common/middleware/s3api/test_s3request.py
@@ -143,7 +143,7 @@ class TestRequest(S3ApiTestCase):
def test_get_response_without_match_ACL_MAP(self):
with self.assertRaises(Exception) as e:
self._test_get_response('POST', req_klass=S3AclRequest)
- self.assertEqual(e.exception.message,
+ self.assertEqual(e.exception.args[0],
'No permission to be checked exists')
def test_get_response_without_duplication_HEAD_request(self):
@@ -215,8 +215,8 @@ class TestRequest(S3ApiTestCase):
s3req = create_s3request_with_param('max-keys', '1' * 30)
with self.assertRaises(InvalidArgument) as result:
s3req.get_validated_param('max-keys', 1)
- self.assertTrue(
- 'not an integer or within integer range' in result.exception.body)
+ self.assertIn(
+ b'not an integer or within integer range', result.exception.body)
self.assertEqual(
result.exception.headers['content-type'], 'application/xml')
@@ -224,8 +224,8 @@ class TestRequest(S3ApiTestCase):
s3req = create_s3request_with_param('max-keys', '-1')
with self.assertRaises(InvalidArgument) as result:
s3req.get_validated_param('max-keys', 1)
- self.assertTrue(
- 'must be an integer between 0 and' in result.exception.body)
+ self.assertIn(
+ b'must be an integer between 0 and', result.exception.body)
self.assertEqual(
result.exception.headers['content-type'], 'application/xml')
@@ -233,8 +233,8 @@ class TestRequest(S3ApiTestCase):
s3req = create_s3request_with_param('max-keys', 'invalid')
with self.assertRaises(InvalidArgument) as result:
s3req.get_validated_param('max-keys', 1)
- self.assertTrue(
- 'not an integer or within integer range' in result.exception.body)
+ self.assertIn(
+ b'not an integer or within integer range', result.exception.body)
self.assertEqual(
result.exception.headers['content-type'], 'application/xml')
@@ -351,7 +351,7 @@ class TestRequest(S3ApiTestCase):
headers={'Authorization': 'AWS test:tester:hmac'})
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '403')
- self.assertEqual(body, '')
+ self.assertEqual(body, b'')
def test_date_header_expired(self):
self.swift.register('HEAD', '/v1/AUTH_test/nojunk', swob.HTTPNotFound,
@@ -363,7 +363,7 @@ class TestRequest(S3ApiTestCase):
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '403')
- self.assertEqual(body, '')
+ self.assertEqual(body, b'')
def test_date_header_with_x_amz_date_valid(self):
self.swift.register('HEAD', '/v1/AUTH_test/nojunk', swob.HTTPNotFound,
@@ -376,7 +376,7 @@ class TestRequest(S3ApiTestCase):
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '404')
- self.assertEqual(body, '')
+ self.assertEqual(body, b'')
def test_date_header_with_x_amz_date_expired(self):
self.swift.register('HEAD', '/v1/AUTH_test/nojunk', swob.HTTPNotFound,
@@ -390,7 +390,7 @@ class TestRequest(S3ApiTestCase):
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '403')
- self.assertEqual(body, '')
+ self.assertEqual(body, b'')
def _test_request_timestamp_sigv4(self, date_header):
# signature v4 here
@@ -428,7 +428,7 @@ class TestRequest(S3ApiTestCase):
def test_request_timestamp_sigv4(self):
access_denied_message = \
- 'AWS authentication requires a valid Date or x-amz-date header'
+ b'AWS authentication requires a valid Date or x-amz-date header'
# normal X-Amz-Date header
date_header = {'X-Amz-Date': self.get_v4_amz_date_header()}
@@ -443,7 +443,7 @@ class TestRequest(S3ApiTestCase):
with self.assertRaises(AccessDenied) as cm:
self._test_request_timestamp_sigv4(date_header)
- self.assertEqual('403 Forbidden', cm.exception.message)
+ self.assertEqual('403 Forbidden', cm.exception.args[0])
self.assertIn(access_denied_message, cm.exception.body)
# mangled Date header
@@ -451,7 +451,7 @@ class TestRequest(S3ApiTestCase):
with self.assertRaises(AccessDenied) as cm:
self._test_request_timestamp_sigv4(date_header)
- self.assertEqual('403 Forbidden', cm.exception.message)
+ self.assertEqual('403 Forbidden', cm.exception.args[0])
self.assertIn(access_denied_message, cm.exception.body)
# Negative timestamp
@@ -459,7 +459,7 @@ class TestRequest(S3ApiTestCase):
with self.assertRaises(AccessDenied) as cm:
self._test_request_timestamp_sigv4(date_header)
- self.assertEqual('403 Forbidden', cm.exception.message)
+ self.assertEqual('403 Forbidden', cm.exception.args[0])
self.assertIn(access_denied_message, cm.exception.body)
# far-past Date header
@@ -467,7 +467,7 @@ class TestRequest(S3ApiTestCase):
with self.assertRaises(AccessDenied) as cm:
self._test_request_timestamp_sigv4(date_header)
- self.assertEqual('403 Forbidden', cm.exception.message)
+ self.assertEqual('403 Forbidden', cm.exception.args[0])
self.assertIn(access_denied_message, cm.exception.body)
# near-future X-Amz-Date header
@@ -481,9 +481,9 @@ class TestRequest(S3ApiTestCase):
with self.assertRaises(RequestTimeTooSkewed) as cm:
self._test_request_timestamp_sigv4(date_header)
- self.assertEqual('403 Forbidden', cm.exception.message)
- self.assertIn('The difference between the request time and the '
- 'current time is too large.', cm.exception.body)
+ self.assertEqual('403 Forbidden', cm.exception.args[0])
+ self.assertIn(b'The difference between the request time and the '
+ b'current time is too large.', cm.exception.body)
def _test_request_timestamp_sigv2(self, date_header):
# signature v4 here
@@ -505,7 +505,7 @@ class TestRequest(S3ApiTestCase):
def test_request_timestamp_sigv2(self):
access_denied_message = \
- 'AWS authentication requires a valid Date or x-amz-date header'
+ b'AWS authentication requires a valid Date or x-amz-date header'
# In v2 format, normal X-Amz-Date header is same
date_header = {'X-Amz-Date': self.get_date_header()}
@@ -520,7 +520,7 @@ class TestRequest(S3ApiTestCase):
with self.assertRaises(AccessDenied) as cm:
self._test_request_timestamp_sigv2(date_header)
- self.assertEqual('403 Forbidden', cm.exception.message)
+ self.assertEqual('403 Forbidden', cm.exception.args[0])
self.assertIn(access_denied_message, cm.exception.body)
# mangled Date header
@@ -528,7 +528,7 @@ class TestRequest(S3ApiTestCase):
with self.assertRaises(AccessDenied) as cm:
self._test_request_timestamp_sigv2(date_header)
- self.assertEqual('403 Forbidden', cm.exception.message)
+ self.assertEqual('403 Forbidden', cm.exception.args[0])
self.assertIn(access_denied_message, cm.exception.body)
# Negative timestamp
@@ -536,7 +536,7 @@ class TestRequest(S3ApiTestCase):
with self.assertRaises(AccessDenied) as cm:
self._test_request_timestamp_sigv2(date_header)
- self.assertEqual('403 Forbidden', cm.exception.message)
+ self.assertEqual('403 Forbidden', cm.exception.args[0])
self.assertIn(access_denied_message, cm.exception.body)
# far-past Date header
@@ -544,7 +544,7 @@ class TestRequest(S3ApiTestCase):
with self.assertRaises(AccessDenied) as cm:
self._test_request_timestamp_sigv2(date_header)
- self.assertEqual('403 Forbidden', cm.exception.message)
+ self.assertEqual('403 Forbidden', cm.exception.args[0])
self.assertIn(access_denied_message, cm.exception.body)
# far-future Date header
@@ -552,9 +552,9 @@ class TestRequest(S3ApiTestCase):
with self.assertRaises(RequestTimeTooSkewed) as cm:
self._test_request_timestamp_sigv2(date_header)
- self.assertEqual('403 Forbidden', cm.exception.message)
- self.assertIn('The difference between the request time and the '
- 'current time is too large.', cm.exception.body)
+ self.assertEqual('403 Forbidden', cm.exception.args[0])
+ self.assertIn(b'The difference between the request time and the '
+ b'current time is too large.', cm.exception.body)
def test_headers_to_sign_sigv4(self):
environ = {
@@ -681,14 +681,14 @@ class TestRequest(S3ApiTestCase):
sigv4_req = SigV4Request(req.environ)
uri = sigv4_req._canonical_uri()
- self.assertEqual(uri, '/')
+ self.assertEqual(uri, b'/')
self.assertEqual(req.environ['PATH_INFO'], '/')
req = Request.blank('/obj1', environ=environ, headers=headers)
sigv4_req = SigV4Request(req.environ)
uri = sigv4_req._canonical_uri()
- self.assertEqual(uri, '/obj1')
+ self.assertEqual(uri, b'/obj1')
self.assertEqual(req.environ['PATH_INFO'], '/obj1')
environ = {
@@ -701,7 +701,7 @@ class TestRequest(S3ApiTestCase):
sigv4_req = SigV4Request(req.environ)
uri = sigv4_req._canonical_uri()
- self.assertEqual(uri, '/')
+ self.assertEqual(uri, b'/')
self.assertEqual(req.environ['PATH_INFO'], '/')
req = Request.blank('/bucket/obj1',
@@ -710,7 +710,7 @@ class TestRequest(S3ApiTestCase):
sigv4_req = SigV4Request(req.environ)
uri = sigv4_req._canonical_uri()
- self.assertEqual(uri, '/bucket/obj1')
+ self.assertEqual(uri, b'/bucket/obj1')
self.assertEqual(req.environ['PATH_INFO'], '/bucket/obj1')
@patch.object(S3Request, '_validate_dates', lambda *a: None)
@@ -724,12 +724,12 @@ class TestRequest(S3ApiTestCase):
'bWq2s1WEIj+Ydj0vQ697zp+IXMU='),
})
sigv2_req = S3Request(req.environ, storage_domain='s3.amazonaws.com')
- expected_sts = '\n'.join([
- 'GET',
- '',
- '',
- 'Tue, 27 Mar 2007 19:36:42 +0000',
- '/johnsmith/photos/puppy.jpg',
+ expected_sts = b'\n'.join([
+ b'GET',
+ b'',
+ b'',
+ b'Tue, 27 Mar 2007 19:36:42 +0000',
+ b'/johnsmith/photos/puppy.jpg',
])
self.assertEqual(expected_sts, sigv2_req._string_to_sign())
self.assertTrue(sigv2_req.check_signature(secret))
@@ -743,12 +743,12 @@ class TestRequest(S3ApiTestCase):
'MyyxeRY7whkBe+bq8fHCL/2kKUg='),
})
sigv2_req = S3Request(req.environ, storage_domain='s3.amazonaws.com')
- expected_sts = '\n'.join([
- 'PUT',
- '',
- 'image/jpeg',
- 'Tue, 27 Mar 2007 21:15:45 +0000',
- '/johnsmith/photos/puppy.jpg',
+ expected_sts = b'\n'.join([
+ b'PUT',
+ b'',
+ b'image/jpeg',
+ b'Tue, 27 Mar 2007 21:15:45 +0000',
+ b'/johnsmith/photos/puppy.jpg',
])
self.assertEqual(expected_sts, sigv2_req._string_to_sign())
self.assertTrue(sigv2_req.check_signature(secret))
@@ -763,12 +763,12 @@ class TestRequest(S3ApiTestCase):
'htDYFYduRNen8P9ZfE/s9SuKy0U='),
})
sigv2_req = S3Request(req.environ, storage_domain='s3.amazonaws.com')
- expected_sts = '\n'.join([
- 'GET',
- '',
- '',
- 'Tue, 27 Mar 2007 19:42:41 +0000',
- '/johnsmith/',
+ expected_sts = b'\n'.join([
+ b'GET',
+ b'',
+ b'',
+ b'Tue, 27 Mar 2007 19:42:41 +0000',
+ b'/johnsmith/',
])
self.assertEqual(expected_sts, sigv2_req._string_to_sign())
self.assertTrue(sigv2_req.check_signature(secret))
@@ -846,7 +846,7 @@ class TestHashingInput(S3ApiTestCase):
self.assertEqual(b'1234', wrapped.read(4))
self.assertEqual(b'56', wrapped.read(2))
# even though the hash matches, there was more data than we expected
- with self.assertRaises(swob.Response) as raised:
+ with self.assertRaises(swob.HTTPException) as raised:
wrapped.read(3)
self.assertEqual(raised.exception.status, '422 Unprocessable Entity')
# the error causes us to close the input
@@ -859,7 +859,7 @@ class TestHashingInput(S3ApiTestCase):
self.assertEqual(b'1234', wrapped.read(4))
self.assertEqual(b'56', wrapped.read(2))
# even though the hash matches, there was more data than we expected
- with self.assertRaises(swob.Response) as raised:
+ with self.assertRaises(swob.HTTPException) as raised:
wrapped.read(4)
self.assertEqual(raised.exception.status, '422 Unprocessable Entity')
self.assertTrue(wrapped._input.closed)
@@ -870,14 +870,14 @@ class TestHashingInput(S3ApiTestCase):
hashlib.md5(raw).hexdigest())
self.assertEqual(b'1234', wrapped.read(4))
self.assertEqual(b'5678', wrapped.read(4))
- with self.assertRaises(swob.Response) as raised:
+ with self.assertRaises(swob.HTTPException) as raised:
wrapped.read(4)
self.assertEqual(raised.exception.status, '422 Unprocessable Entity')
self.assertTrue(wrapped._input.closed)
def test_empty_bad_hash(self):
wrapped = HashingInput(BytesIO(b''), 0, hashlib.sha256, 'nope')
- with self.assertRaises(swob.Response) as raised:
+ with self.assertRaises(swob.HTTPException) as raised:
wrapped.read(3)
self.assertEqual(raised.exception.status, '422 Unprocessable Entity')
# the error causes us to close the input
diff --git a/test/unit/common/middleware/s3api/test_service.py b/test/unit/common/middleware/s3api/test_service.py
index 2f8286ba8b..f32046052a 100644
--- a/test/unit/common/middleware/s3api/test_service.py
+++ b/test/unit/common/middleware/s3api/test_service.py
@@ -135,7 +135,7 @@ class TestS3ApiService(S3ApiTestCase):
self.assertEqual(len(names), len(expected))
for i in expected:
- self.assertTrue(i[0] in names)
+ self.assertIn(i[0], names)
def _test_service_GET_for_check_bucket_owner(self, buckets):
self.s3api.conf.check_bucket_owner = True
diff --git a/test/unit/common/test_internal_client.py b/test/unit/common/test_internal_client.py
index 22f88f25ac..96d02401fa 100644
--- a/test/unit/common/test_internal_client.py
+++ b/test/unit/common/test_internal_client.py
@@ -1289,7 +1289,7 @@ class TestInternalClient(unittest.TestCase):
def fake_app(self, env, start_response):
start_response('404 Not Found', [])
- return ['one\ntwo\nthree']
+ return [b'one\ntwo\nthree']
client = InternalClient()
lines = []
diff --git a/test/unit/common/test_swob.py b/test/unit/common/test_swob.py
index a30994ab3a..9ebf95c3ce 100644
--- a/test/unit/common/test_swob.py
+++ b/test/unit/common/test_swob.py
@@ -670,7 +670,7 @@ class TestRequest(unittest.TestCase):
def test_app(environ, start_response):
start_response('401 Unauthorized', [])
- return ['hi']
+ return [b'hi']
# Request environment contains valid account in path
req = swift.common.swob.Request.blank('/v1/account-name')
@@ -692,7 +692,7 @@ class TestRequest(unittest.TestCase):
def test_app(environ, start_response):
start_response('401 Unauthorized', [])
- return ['hi']
+ return [b'hi']
# Request environment contains bad path
req = swift.common.swob.Request.blank('/random')
@@ -706,7 +706,7 @@ class TestRequest(unittest.TestCase):
def test_app(environ, start_response):
start_response('401 Unauthorized', [])
- return ['no creds in request']
+ return [b'no creds in request']
# Request to get token
req = swift.common.swob.Request.blank('/v1.0/auth')
@@ -729,7 +729,7 @@ class TestRequest(unittest.TestCase):
def test_app(environ, start_response):
start_response('401 Unauthorized', {
'Www-Authenticate': 'Me realm="whatever"'})
- return ['no creds in request']
+ return [b'no creds in request']
# Auth middleware sets own Www-Authenticate
req = swift.common.swob.Request.blank('/auth/v1.0')
@@ -743,7 +743,7 @@ class TestRequest(unittest.TestCase):
def test_app(environ, start_response):
start_response('401 Unauthorized', [])
- return ['hi']
+ return [b'hi']
hacker = 'account-name\n\nfoo
' # url injection test
quoted_hacker = quote(hacker)
@@ -766,7 +766,7 @@ class TestRequest(unittest.TestCase):
# Other status codes should not have WWW-Authenticate in response
def test_app(environ, start_response):
start_response('200 OK', [])
- return ['hi']
+ return [b'hi']
req = swift.common.swob.Request.blank('/')
resp = req.get_response(test_app)
@@ -1758,7 +1758,7 @@ class TestConditionalIfMatch(unittest.TestCase):
def fake_app_404(environ, start_response):
start_response('404 Not Found', [])
- return ['hi']
+ return [b'hi']
req = swift.common.swob.Request.blank(
'/', headers={'If-Match': '*'})
diff --git a/tox.ini b/tox.ini
index 05f64112d6..9147ec8bb3 100644
--- a/tox.ini
+++ b/tox.ini
@@ -40,7 +40,7 @@ commands =
test/unit/account \
test/unit/cli \
test/unit/common/middleware/crypto \
- test/unit/common/middleware/s3api/test_s3token.py \
+ test/unit/common/middleware/s3api/ \
test/unit/common/middleware/test_account_quotas.py \
test/unit/common/middleware/test_acl.py \
test/unit/common/middleware/test_catch_errors.py \