openstack/swift
Code Issues Proposed changes

py3: port s3api

Browse Source 
Drive-by: When passing a list or tuple to swob.Response as an app_iter,
check that it's full of byte strings.

Change-Id: Ifc35aacb2e45004f74c871f08ff3c52bc57c1463
This commit is contained in:
Tim Burke 2019-04-09 16:47:20 -07:00
parent f55167a735
commit 3a9f3f8419
23 changed files with 308 additions and 230 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
swift/common/middleware/s3api/controllers/bucket.py
View File

@ -18,6 +18,7 @@ from base64 import standard_b64decode as b64decode
from six.moves.urllib.parse import quote from six.moves.urllib.parse import quote
from swift.common import swob
from swift.common.http import HTTP_OK from swift.common.http import HTTP_OK
from swift.common.utils import json, public, config_true_value from swift.common.utils import json, public, config_true_value
@ -66,8 +67,9 @@ class BucketController(Controller):
segments = json.loads(resp.body) segments = json.loads(resp.body)
for seg in segments: for seg in segments:
try: try:
req.get_response(self.app, 'DELETE', container, req.get_response(
seg['name'].encode('utf8')) self.app, 'DELETE', container,
swob.bytes_to_wsgi(seg['name'].encode('utf8')))
except NoSuchKey: except NoSuchKey:
pass pass
except InternalError: except InternalError:

25
swift/common/middleware/s3api/controllers/multi_upload.py
View File

@ -59,11 +59,14 @@ Static Large Object when the multipart upload is completed.
""" """
import binascii
from hashlib import md5 from hashlib import md5
import os import os
import re import re
import time import time
import six
from swift.common.swob import Range from swift.common.swob import Range
from swift.common.utils import json, public, reiterate from swift.common.utils import json, public, reiterate
from swift.common.db import utf8encode from swift.common.db import utf8encode
@ -222,6 +225,7 @@ class UploadsController(Controller):
:return (non_delimited_uploads, common_prefixes) :return (non_delimited_uploads, common_prefixes)
""" """
if six.PY2:
(prefix, delimiter) = \ (prefix, delimiter) = \
utf8encode(prefix, delimiter) utf8encode(prefix, delimiter)
non_delimited_uploads = [] non_delimited_uploads = []
@ -440,7 +444,7 @@ class UploadController(Controller):
# If the caller requested a list starting at a specific part number, # If the caller requested a list starting at a specific part number,
# construct a sub-set of the object list. # construct a sub-set of the object list.
objList = filter(filter_part_num_marker, objects) objList = [obj for obj in objects if filter_part_num_marker(obj)]
# pylint: disable-msg=E1103 # pylint: disable-msg=E1103
objList.sort(key=lambda o: int(o['name'].split('/')[-1])) objList.sort(key=lambda o: int(o['name'].split('/')[-1]))
@ -603,7 +607,7 @@ class UploadController(Controller):
'path': '/%s/%s/%s/%d' % ( 'path': '/%s/%s/%s/%d' % (
container, req.object_name, upload_id, part_number), container, req.object_name, upload_id, part_number),
'etag': etag}) 'etag': etag})
s3_etag_hasher.update(etag.decode('hex')) s3_etag_hasher.update(binascii.a2b_hex(etag))
except (XMLSyntaxError, DocumentInvalid): except (XMLSyntaxError, DocumentInvalid):
# NB: our schema definitions catch uploads with no parts here # NB: our schema definitions catch uploads with no parts here
raise MalformedXML() raise MalformedXML()
@ -661,8 +665,8 @@ class UploadController(Controller):
# ceph-s3tests happy # ceph-s3tests happy
continue continue
if not yielded_anything: if not yielded_anything:
yield ('<?xml version="1.0" ' yield (b'<?xml version="1.0" '
'encoding="UTF-8"?>\n') b'encoding="UTF-8"?>\n')
yielded_anything = True yielded_anything = True
yield chunk yield chunk
continue continue
@ -708,8 +712,13 @@ class UploadController(Controller):
# in detail, https://github.com/boto/boto/pull/3513 # in detail, https://github.com/boto/boto/pull/3513
parsed_url = urlparse(req.host_url) parsed_url = urlparse(req.host_url)
host_url = '%s://%s' % (parsed_url.scheme, parsed_url.hostname) host_url = '%s://%s' % (parsed_url.scheme, parsed_url.hostname)
if parsed_url.port: # Why are we doing our own port parsing? Because py3 decided
host_url += ':%s' % parsed_url.port # to start raising ValueErrors on access after parsing such
# an invalid port
netloc = parsed_url.netloc.split('@')[-1].split(']')[-1]
if ':' in netloc:
port = netloc.split(':', 2)[1]
host_url += ':%s' % port
SubElement(result_elem, 'Location').text = host_url + req.path SubElement(result_elem, 'Location').text = host_url + req.path
SubElement(result_elem, 'Bucket').text = req.container_name SubElement(result_elem, 'Bucket').text = req.container_name
@ -717,13 +726,13 @@ class UploadController(Controller):
SubElement(result_elem, 'ETag').text = '"%s"' % s3_etag SubElement(result_elem, 'ETag').text = '"%s"' % s3_etag
resp.headers.pop('ETag', None) resp.headers.pop('ETag', None)
if yielded_anything: if yielded_anything:
yield '\n' yield b'\n'
yield tostring(result_elem, yield tostring(result_elem,
xml_declaration=not yielded_anything) xml_declaration=not yielded_anything)
except ErrorResponse as err_resp: except ErrorResponse as err_resp:
if yielded_anything: if yielded_anything:
err_resp.xml_declaration = False err_resp.xml_declaration = False
yield '\n' yield b'\n'
else: else:
# Oh good, we can still change HTTP status code, too! # Oh good, we can still change HTTP status code, too!
resp.status = err_resp.status resp.status = err_resp.status

2
swift/common/middleware/s3api/controllers/obj.py
View File

@ -156,7 +156,7 @@ class ObjectController(Controller):
for chunk in resp.app_iter: for chunk in resp.app_iter:
pass # drain the bulk-deleter response pass # drain the bulk-deleter response
resp.status = HTTP_NO_CONTENT resp.status = HTTP_NO_CONTENT
resp.body = '' resp.body = b''
except NoSuchKey: except NoSuchKey:
# expect to raise NoSuchBucket when the bucket doesn't exist # expect to raise NoSuchBucket when the bucket doesn't exist
req.get_container_info(self.app) req.get_container_info(self.app)

2
swift/common/middleware/s3api/etree.py
View File

@ -120,7 +120,9 @@ class _Element(lxml.etree.ElementBase):
""" """
utf-8 wrapper property of lxml.etree.Element.text utf-8 wrapper property of lxml.etree.Element.text
""" """
if six.PY2:
return utf8encode(lxml.etree.ElementBase.text.__get__(self)) return utf8encode(lxml.etree.ElementBase.text.__get__(self))
return lxml.etree.ElementBase.text.__get__(self)
@text.setter @text.setter
def text(self, value): def text(self, value):

4
swift/common/middleware/s3api/s3api.py
View File

@ -167,7 +167,7 @@ class ListingEtagMiddleware(object):
ctx._response_exc_info) ctx._response_exc_info)
return [body] return [body]
body = json.dumps(listing) body = json.dumps(listing).encode('ascii')
ctx._response_headers[cl_index] = ( ctx._response_headers[cl_index] = (
ctx._response_headers[cl_index][0], ctx._response_headers[cl_index][0],
str(len(body)), str(len(body)),
@ -237,7 +237,7 @@ class S3ApiMiddleware(object):
resp = err_resp resp = err_resp
except Exception as e: except Exception as e:
self.logger.exception(e) self.logger.exception(e)
resp = InternalError(reason=e) resp = InternalError(reason=str(e))
if isinstance(resp, S3ResponseBase) and 'swift.trans_id' in env: if isinstance(resp, S3ResponseBase) and 'swift.trans_id' in env:
resp.headers['x-amz-id-2'] = env['swift.trans_id'] resp.headers['x-amz-id-2'] = env['swift.trans_id']

80
swift/common/middleware/s3api/s3request.py
View File

@ -14,6 +14,7 @@
# limitations under the License. # limitations under the License.
import base64 import base64
import binascii
from collections import defaultdict, OrderedDict from collections import defaultdict, OrderedDict
from email.header import Header from email.header import Header
from hashlib import sha1, sha256, md5 from hashlib import sha1, sha256, md5
@ -127,7 +128,8 @@ class HashingInput(object):
chunk = self._input.read(size) chunk = self._input.read(size)
self._hasher.update(chunk) self._hasher.update(chunk)
self._to_read -= len(chunk) self._to_read -= len(chunk)
if self._to_read < 0 or (size > len(chunk) and self._to_read) or ( short_read = bool(chunk) if size is None else (len(chunk) < size)
if self._to_read < 0 or (short_read and self._to_read) or (
self._to_read == 0 and self._to_read == 0 and
self._hasher.hexdigest() != self._expected): self._hasher.hexdigest() != self._expected):
self.close() self.close()
@ -149,10 +151,10 @@ class SigV4Mixin(object):
def check_signature(self, secret): def check_signature(self, secret):
secret = utf8encode(secret) secret = utf8encode(secret)
user_signature = self.signature user_signature = self.signature
derived_secret = 'AWS4' + secret derived_secret = b'AWS4' + secret
for scope_piece in self.scope.values(): for scope_piece in self.scope.values():
derived_secret = hmac.new( derived_secret = hmac.new(
derived_secret, scope_piece, sha256).digest() derived_secret, scope_piece.encode('utf8'), sha256).digest()
valid_signature = hmac.new( valid_signature = hmac.new(
derived_secret, self.string_to_sign, sha256).hexdigest() derived_secret, self.string_to_sign, sha256).hexdigest()
return user_signature == valid_signature return user_signature == valid_signature
@ -331,10 +333,10 @@ class SigV4Mixin(object):
def _canonical_query_string(self): def _canonical_query_string(self):
return '&'.join( return '&'.join(
'%s=%s' % (quote(key, safe='-_.~'), '%s=%s' % (swob.wsgi_quote(key, safe='-_.~'),
quote(value, safe='-_.~')) swob.wsgi_quote(value, safe='-_.~'))
for key, value in sorted(self.params.items()) for key, value in sorted(self.params.items())
if key not in ('Signature', 'X-Amz-Signature')) if key not in ('Signature', 'X-Amz-Signature')).encode('ascii')
def _headers_to_sign(self): def _headers_to_sign(self):
""" """
@ -383,7 +385,7 @@ class SigV4Mixin(object):
""" """
It won't require bucket name in canonical_uri for v4. It won't require bucket name in canonical_uri for v4.
""" """
return self.environ.get('RAW_PATH_INFO', self.path) return swob.wsgi_to_bytes(self.environ.get('RAW_PATH_INFO', self.path))
def _canonical_request(self): def _canonical_request(self):
# prepare 'canonical_request' # prepare 'canonical_request'
@ -401,7 +403,7 @@ class SigV4Mixin(object):
# #
# 1. Add verb like: GET # 1. Add verb like: GET
cr = [self.method.upper()] cr = [swob.wsgi_to_bytes(self.method.upper())]
# 2. Add path like: / # 2. Add path like: /
path = self._canonical_uri() path = self._canonical_uri()
@ -415,12 +417,12 @@ class SigV4Mixin(object):
# host:iam.amazonaws.com # host:iam.amazonaws.com
# x-amz-date:20150830T123600Z # x-amz-date:20150830T123600Z
headers_to_sign = self._headers_to_sign() headers_to_sign = self._headers_to_sign()
cr.append(''.join('%s:%s\n' % (key, value) cr.append(b''.join(swob.wsgi_to_bytes('%s:%s\n' % (key, value))
for key, value in headers_to_sign)) for key, value in headers_to_sign))
# 5. Add signed headers into canonical request like # 5. Add signed headers into canonical request like
# content-type;host;x-amz-date # content-type;host;x-amz-date
cr.append(';'.join(k for k, v in headers_to_sign)) cr.append(b';'.join(swob.wsgi_to_bytes(k) for k, v in headers_to_sign))
# 6. Add payload string at the tail # 6. Add payload string at the tail
if 'X-Amz-Credential' in self.params: if 'X-Amz-Credential' in self.params:
@ -446,8 +448,8 @@ class SigV4Mixin(object):
# else, not provided -- Swift will kick out a 411 Length Required # else, not provided -- Swift will kick out a 411 Length Required
# which will get translated back to a S3-style response in # which will get translated back to a S3-style response in
# S3Request._swift_error_codes # S3Request._swift_error_codes
cr.append(hashed_payload) cr.append(swob.wsgi_to_bytes(hashed_payload))
return '\n'.join(cr).encode('utf-8') return b'\n'.join(cr)
@property @property
def scope(self): def scope(self):
@ -462,10 +464,11 @@ class SigV4Mixin(object):
""" """
Create 'StringToSign' value in Amazon terminology for v4. Create 'StringToSign' value in Amazon terminology for v4.
""" """
return '\n'.join(['AWS4-HMAC-SHA256', return b'\n'.join([
self.timestamp.amz_date_format, b'AWS4-HMAC-SHA256',
'/'.join(self.scope.values()), self.timestamp.amz_date_format.encode('ascii'),
sha256(self._canonical_request()).hexdigest()]) '/'.join(self.scope.values()).encode('utf8'),
sha256(self._canonical_request()).hexdigest().encode('ascii')])
def signature_does_not_match_kwargs(self): def signature_does_not_match_kwargs(self):
kwargs = super(SigV4Mixin, self).signature_does_not_match_kwargs() kwargs = super(SigV4Mixin, self).signature_does_not_match_kwargs()
@ -473,7 +476,7 @@ class SigV4Mixin(object):
kwargs.update({ kwargs.update({
'canonical_request': cr, 'canonical_request': cr,
'canonical_request_bytes': ' '.join( 'canonical_request_bytes': ' '.join(
format(ord(c), '02x') for c in cr), format(ord(c), '02x') for c in cr.decode('latin1')),
}) })
return kwargs return kwargs
@ -545,6 +548,8 @@ class S3Request(swob.Request):
user_signature = self.signature user_signature = self.signature
valid_signature = base64.b64encode(hmac.new( valid_signature = base64.b64encode(hmac.new(
secret, self.string_to_sign, sha1).digest()).strip() secret, self.string_to_sign, sha1).digest()).strip()
if not six.PY2:
valid_signature = valid_signature.decode('ascii')
return user_signature == valid_signature return user_signature == valid_signature
@property @property
@ -613,7 +618,7 @@ class S3Request(swob.Request):
return None return None
def _parse_uri(self): def _parse_uri(self):
if not check_utf8(self.environ['PATH_INFO']): if not check_utf8(swob.wsgi_to_str(self.environ['PATH_INFO'])):
raise InvalidURI(self.path) raise InvalidURI(self.path)
if self.bucket_in_host: if self.bucket_in_host:
@ -739,8 +744,10 @@ class S3Request(swob.Request):
# Non-base64-alphabet characters in value. # Non-base64-alphabet characters in value.
raise InvalidDigest(content_md5=value) raise InvalidDigest(content_md5=value)
try: try:
self.headers['ETag'] = value.decode('base64').encode('hex') self.headers['ETag'] = binascii.b2a_hex(
except Exception: binascii.a2b_base64(value))
except binascii.error:
# incorrect padding, most likely
raise InvalidDigest(content_md5=value) raise InvalidDigest(content_md5=value)
if len(self.headers['ETag']) != 32: if len(self.headers['ETag']) != 32:
@ -825,10 +832,11 @@ class S3Request(swob.Request):
'functionality that is not implemented', 'functionality that is not implemented',
header='Transfer-Encoding') header='Transfer-Encoding')
if self.message_length() > max_length: ml = self.message_length()
if ml and ml > max_length:
raise MalformedXML() raise MalformedXML()
if te or self.message_length(): if te or ml:
# Limit the read similar to how SLO handles manifests # Limit the read similar to how SLO handles manifests
body = self.body_file.read(max_length) body = self.body_file.read(max_length)
else: else:
@ -843,7 +851,7 @@ class S3Request(swob.Request):
raise InvalidRequest('Missing required header for this request: ' raise InvalidRequest('Missing required header for this request: '
'Content-MD5') 'Content-MD5')
digest = md5(body).digest().encode('base64').strip() digest = base64.b64encode(md5(body).digest()).strip().decode('ascii')
if self.environ['HTTP_CONTENT_MD5'] != digest: if self.environ['HTTP_CONTENT_MD5'] != digest:
raise BadDigest(content_md5=self.environ['HTTP_CONTENT_MD5']) raise BadDigest(content_md5=self.environ['HTTP_CONTENT_MD5'])
@ -927,9 +935,10 @@ class S3Request(swob.Request):
""" """
amz_headers = {} amz_headers = {}
buf = [self.method, buf = [swob.wsgi_to_bytes(wsgi_str) for wsgi_str in [
self.method,
_header_strip(self.headers.get('Content-MD5')) or '', _header_strip(self.headers.get('Content-MD5')) or '',
_header_strip(self.headers.get('Content-Type')) or ''] _header_strip(self.headers.get('Content-Type')) or '']]
if 'headers_raw' in self.environ: # eventlet >= 0.19.0 if 'headers_raw' in self.environ: # eventlet >= 0.19.0
# See https://github.com/eventlet/eventlet/commit/67ec999 # See https://github.com/eventlet/eventlet/commit/67ec999
@ -948,18 +957,18 @@ class S3Request(swob.Request):
if self._is_header_auth: if self._is_header_auth:
if 'x-amz-date' in amz_headers: if 'x-amz-date' in amz_headers:
buf.append('') buf.append(b'')
elif 'Date' in self.headers: elif 'Date' in self.headers:
buf.append(self.headers['Date']) buf.append(swob.wsgi_to_bytes(self.headers['Date']))
elif self._is_query_auth: elif self._is_query_auth:
buf.append(self.params['Expires']) buf.append(swob.wsgi_to_bytes(self.params['Expires']))
else: else:
# Should have already raised NotS3Request in _parse_auth_info, # Should have already raised NotS3Request in _parse_auth_info,
# but as a sanity check... # but as a sanity check...
raise AccessDenied() raise AccessDenied()
for key, value in sorted(amz_headers.items()): for key, value in sorted(amz_headers.items()):
buf.append("%s:%s" % (key, value)) buf.append(swob.wsgi_to_bytes("%s:%s" % (key, value)))
path = self._canonical_uri() path = self._canonical_uri()
if self.query_string: if self.query_string:
@ -971,10 +980,10 @@ class S3Request(swob.Request):
if key in ALLOWED_SUB_RESOURCES: if key in ALLOWED_SUB_RESOURCES:
params.append('%s=%s' % (key, value) if value else key) params.append('%s=%s' % (key, value) if value else key)
if params: if params:
buf.append('%s?%s' % (path, '&'.join(params))) buf.append(swob.wsgi_to_bytes('%s?%s' % (path, '&'.join(params))))
else: else:
buf.append(path) buf.append(swob.wsgi_to_bytes(path))
return '\n'.join(buf) return b'\n'.join(buf)
def signature_does_not_match_kwargs(self): def signature_does_not_match_kwargs(self):
return { return {
@ -982,7 +991,8 @@ class S3Request(swob.Request):
'string_to_sign': self.string_to_sign, 'string_to_sign': self.string_to_sign,
'signature_provided': self.signature, 'signature_provided': self.signature,
'string_to_sign_bytes': ' '.join( 'string_to_sign_bytes': ' '.join(
format(ord(c), '02x') for c in self.string_to_sign), format(ord(c), '02x')
for c in self.string_to_sign.decode('latin1')),
} }
@property @property
@ -1320,7 +1330,6 @@ class S3Request(swob.Request):
# reuse account and tokens # reuse account and tokens
_, self.account, _ = split_path(sw_resp.environ['PATH_INFO'], _, self.account, _ = split_path(sw_resp.environ['PATH_INFO'],
2, 3, True) 2, 3, True)
self.account = utf8encode(self.account)
resp = S3Response.from_swift_resp(sw_resp) resp = S3Response.from_swift_resp(sw_resp)
status = resp.status_int # pylint: disable-msg=E1101 status = resp.status_int # pylint: disable-msg=E1101
@ -1354,7 +1363,7 @@ class S3Request(swob.Request):
raise err_resp() raise err_resp()
if status == HTTP_BAD_REQUEST: if status == HTTP_BAD_REQUEST:
raise BadSwiftRequest(err_msg) raise BadSwiftRequest(err_msg.decode('utf8'))
if status == HTTP_UNAUTHORIZED: if status == HTTP_UNAUTHORIZED:
raise SignatureDoesNotMatch( raise SignatureDoesNotMatch(
**self.signature_does_not_match_kwargs()) **self.signature_does_not_match_kwargs())
@ -1487,7 +1496,6 @@ class S3AclRequest(S3Request):
_, self.account, _ = split_path(sw_resp.environ['PATH_INFO'], _, self.account, _ = split_path(sw_resp.environ['PATH_INFO'],
2, 3, True) 2, 3, True)
self.account = utf8encode(self.account)
if 'HTTP_X_USER_NAME' in sw_resp.environ: if 'HTTP_X_USER_NAME' in sw_resp.environ:
# keystone # keystone

4
swift/common/middleware/s3api/s3response.py
View File

@ -243,8 +243,10 @@ class ErrorResponse(S3ResponseBase, swob.HTTPException):
if isinstance(value, (dict, MutableMapping)): if isinstance(value, (dict, MutableMapping)):
self._dict_to_etree(elem, value) self._dict_to_etree(elem, value)
else: else:
if isinstance(value, (int, float, bool)):
value = str(value)
try: try:
elem.text = str(value) elem.text = value
except ValueError: except ValueError:
# We set an invalid string for XML. # We set an invalid string for XML.
elem.text = '(invalid string)' elem.text = '(invalid string)'

14
swift/common/middleware/s3api/subresource.py
View File

@ -43,6 +43,8 @@ http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html
""" """
from functools import partial from functools import partial
import six
from swift.common.utils import json from swift.common.utils import json
from swift.common.middleware.s3api.s3response import InvalidArgument, \ from swift.common.middleware.s3api.s3response import InvalidArgument, \
@ -218,6 +220,11 @@ class User(Grantee):
def __str__(self): def __str__(self):
return self.display_name return self.display_name
def __lt__(self, other):
if not isinstance(other, User):
return NotImplemented
return self.id < other.id
class Owner(object): class Owner(object):
""" """
@ -415,9 +422,14 @@ class ACL(object):
self.s3_acl = s3_acl self.s3_acl = s3_acl
self.allow_no_owner = allow_no_owner self.allow_no_owner = allow_no_owner
def __repr__(self): def __bytes__(self):
return tostring(self.elem()) return tostring(self.elem())
def __repr__(self):
if six.PY2:
return self.__bytes__()
return self.__bytes__().decode('utf8')
@classmethod @classmethod
def from_elem(cls, elem, s3_acl=False, allow_no_owner=False): def from_elem(cls, elem, s3_acl=False, allow_no_owner=False):
""" """

10
swift/common/swob.py
View File

@ -309,15 +309,15 @@ def str_to_wsgi(native_str):
return bytes_to_wsgi(native_str.encode('utf8', errors='surrogateescape')) return bytes_to_wsgi(native_str.encode('utf8', errors='surrogateescape'))
def wsgi_quote(wsgi_str): def wsgi_quote(wsgi_str, safe='/'):
if six.PY2: if six.PY2:
if not isinstance(wsgi_str, bytes): if not isinstance(wsgi_str, bytes):
raise TypeError('Expected a WSGI string; got %r' % wsgi_str) raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
return urllib.parse.quote(wsgi_str) return urllib.parse.quote(wsgi_str, safe=safe)
if not isinstance(wsgi_str, str) or any(ord(x) > 255 for x in wsgi_str): if not isinstance(wsgi_str, str) or any(ord(x) > 255 for x in wsgi_str):
raise TypeError('Expected a WSGI string; got %r' % wsgi_str) raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
return urllib.parse.quote(wsgi_str, encoding='latin-1') return urllib.parse.quote(wsgi_str, safe=safe, encoding='latin-1')
def wsgi_unquote(wsgi_str): def wsgi_unquote(wsgi_str):
@ -478,6 +478,10 @@ def _resp_app_iter_property():
def setter(self, value): def setter(self, value):
if isinstance(value, (list, tuple)): if isinstance(value, (list, tuple)):
for i, item in enumerate(value):
if not isinstance(item, bytes):
raise TypeError('WSGI responses must be bytes; '
'got %s for item %d' % (type(item), i))
self.content_length = sum(map(len, value)) self.content_length = sum(map(len, value))
elif value is not None: elif value is not None:
self.content_length = None self.content_length = None

4
test/unit/common/middleware/s3api/helpers.py
View File

@ -165,12 +165,16 @@ class FakeSwift(object):
# keep old sysmeta for s3acl # keep old sysmeta for s3acl
headers.update({key: value}) headers.update({key: value})
if body is not None and not isinstance(body, (bytes, list)):
body = body.encode('utf8')
self._responses[(method, path)] = (response_class, headers, body) self._responses[(method, path)] = (response_class, headers, body)
def register_unconditionally(self, method, path, response_class, headers, def register_unconditionally(self, method, path, response_class, headers,
body): body):
# register() keeps old sysmeta around, but # register() keeps old sysmeta around, but
# register_unconditionally() keeps nothing. # register_unconditionally() keeps nothing.
if body is not None and not isinstance(body, bytes):
body = body.encode('utf8')
self._responses[(method, path)] = (response_class, headers, body) self._responses[(method, path)] = (response_class, headers, body)
def clear_calls(self): def clear_calls(self):

14
test/unit/common/middleware/s3api/test_acl.py
View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import base64
import unittest import unittest
import mock import mock
@ -131,8 +132,8 @@ class TestS3ApiAcl(S3ApiTestCase):
'UnexpectedContent') 'UnexpectedContent')
def _test_put_no_body(self, use_content_length=False, def _test_put_no_body(self, use_content_length=False,
use_transfer_encoding=False, string_to_md5=''): use_transfer_encoding=False, string_to_md5=b''):
content_md5 = md5(string_to_md5).digest().encode('base64').strip() content_md5 = base64.b64encode(md5(string_to_md5).digest()).strip()
with UnreadableInput(self) as fake_input: with UnreadableInput(self) as fake_input:
req = Request.blank( req = Request.blank(
'/bucket?acl', '/bucket?acl',
@ -153,16 +154,17 @@ class TestS3ApiAcl(S3ApiTestCase):
self.assertEqual(self._get_error_code(body), 'MissingSecurityHeader') self.assertEqual(self._get_error_code(body), 'MissingSecurityHeader')
self.assertEqual(self._get_error_message(body), self.assertEqual(self._get_error_message(body),
'Your request was missing a required header.') 'Your request was missing a required header.')
self.assertIn('<MissingHeaderName>x-amz-acl</MissingHeaderName>', body) self.assertIn(b'<MissingHeaderName>x-amz-acl</MissingHeaderName>',
body)
@s3acl @s3acl
def test_bucket_fails_with_neither_acl_header_nor_xml_PUT(self): def test_bucket_fails_with_neither_acl_header_nor_xml_PUT(self):
self._test_put_no_body() self._test_put_no_body()
self._test_put_no_body(string_to_md5='test') self._test_put_no_body(string_to_md5=b'test')
self._test_put_no_body(use_content_length=True) self._test_put_no_body(use_content_length=True)
self._test_put_no_body(use_content_length=True, string_to_md5='test') self._test_put_no_body(use_content_length=True, string_to_md5=b'test')
self._test_put_no_body(use_transfer_encoding=True) self._test_put_no_body(use_transfer_encoding=True)
self._test_put_no_body(use_transfer_encoding=True, string_to_md5='zz') self._test_put_no_body(use_transfer_encoding=True, string_to_md5=b'zz')
def test_object_acl_GET(self): def test_object_acl_GET(self):
req = Request.blank('/bucket/object?acl', req = Request.blank('/bucket/object?acl',

19
test/unit/common/middleware/s3api/test_bucket.py
View File

@ -17,6 +17,7 @@ import unittest
import cgi import cgi
import mock import mock
import six
from six.moves.urllib.parse import quote from six.moves.urllib.parse import quote
from swift.common import swob from swift.common import swob
@ -62,7 +63,8 @@ class TestS3ApiBucket(S3ApiTestCase):
for name, _, _, _ in self.objects: for name, _, _, _ in self.objects:
self.swift.register( self.swift.register(
'DELETE', 'DELETE',
'/v1/AUTH_test/bucket+segments/' + name.encode('utf-8'), '/v1/AUTH_test/bucket+segments/' +
swob.bytes_to_wsgi(name.encode('utf-8')),
swob.HTTPNoContent, {}, json.dumps([])) swob.HTTPNoContent, {}, json.dumps([]))
self.swift.register( self.swift.register(
'GET', 'GET',
@ -118,7 +120,7 @@ class TestS3ApiBucket(S3ApiTestCase):
'Date': self.get_date_header()}) 'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req) status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '404') self.assertEqual(status.split()[0], '404')
self.assertEqual(body, '') # sanity self.assertEqual(body, b'') # sanity
def test_bucket_HEAD_slash(self): def test_bucket_HEAD_slash(self):
req = Request.blank('/junk/', req = Request.blank('/junk/',
@ -168,7 +170,8 @@ class TestS3ApiBucket(S3ApiTestCase):
self.assertEqual('2011-01-05T02:19:14.275Z', self.assertEqual('2011-01-05T02:19:14.275Z',
o.find('./LastModified').text) o.find('./LastModified').text)
self.assertEqual(items, [ self.assertEqual(items, [
(i[0].encode('utf-8'), '"0-N"' if i[0] == 'slo' else '"0"') (i[0].encode('utf-8') if six.PY2 else i[0],
'"0-N"' if i[0] == 'slo' else '"0"')
for i in self.objects]) for i in self.objects])
def test_bucket_GET_url_encoded(self): def test_bucket_GET_url_encoded(self):
@ -519,8 +522,12 @@ class TestS3ApiBucket(S3ApiTestCase):
self.assertEqual(elem.findall('./DeleteMarker'), []) self.assertEqual(elem.findall('./DeleteMarker'), [])
versions = elem.findall('./Version') versions = elem.findall('./Version')
objects = list(self.objects) objects = list(self.objects)
if six.PY2:
self.assertEqual([v.find('./Key').text for v in versions], self.assertEqual([v.find('./Key').text for v in versions],
[v[0].encode('utf-8') for v in objects]) [v[0].encode('utf-8') for v in objects])
else:
self.assertEqual([v.find('./Key').text for v in versions],
[v[0] for v in objects])
self.assertEqual([v.find('./IsLatest').text for v in versions], self.assertEqual([v.find('./IsLatest').text for v in versions],
['true' for v in objects]) ['true' for v in objects])
self.assertEqual([v.find('./VersionId').text for v in versions], self.assertEqual([v.find('./VersionId').text for v in versions],
@ -598,7 +605,7 @@ class TestS3ApiBucket(S3ApiTestCase):
headers={'Authorization': 'AWS test:tester:hmac', headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()}) 'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req) status, headers, body = self.call_s3api(req)
self.assertEqual(body, '') self.assertEqual(body, b'')
self.assertEqual(status.split()[0], '200') self.assertEqual(status.split()[0], '200')
self.assertEqual(headers['Location'], '/bucket') self.assertEqual(headers['Location'], '/bucket')
@ -610,7 +617,7 @@ class TestS3ApiBucket(S3ApiTestCase):
'Date': self.get_date_header(), 'Date': self.get_date_header(),
'Transfer-Encoding': 'chunked'}) 'Transfer-Encoding': 'chunked'})
status, headers, body = self.call_s3api(req) status, headers, body = self.call_s3api(req)
self.assertEqual(body, '') self.assertEqual(body, b'')
self.assertEqual(status.split()[0], '200') self.assertEqual(status.split()[0], '200')
self.assertEqual(headers['Location'], '/bucket') self.assertEqual(headers['Location'], '/bucket')
@ -622,7 +629,7 @@ class TestS3ApiBucket(S3ApiTestCase):
headers={'Authorization': 'AWS test:tester:hmac', headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()}) 'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req) status, headers, body = self.call_s3api(req)
self.assertEqual(body, '') self.assertEqual(body, b'')
self.assertEqual(status.split()[0], '200') self.assertEqual(status.split()[0], '200')
self.assertEqual(headers['Location'], '/bucket') self.assertEqual(headers['Location'], '/bucket')

15
test/unit/common/middleware/s3api/test_etree.py
View File

@ -15,6 +15,8 @@
import unittest import unittest
import six
from swift.common.middleware.s3api import etree from swift.common.middleware.s3api import etree
@ -58,15 +60,18 @@ class TestS3ApiEtree(unittest.TestCase):
sub.text = '\xef\xbc\xa1' sub.text = '\xef\xbc\xa1'
self.assertTrue(isinstance(sub.text, str)) self.assertTrue(isinstance(sub.text, str))
xml_string = etree.tostring(elem) xml_string = etree.tostring(elem)
self.assertTrue(isinstance(xml_string, str)) self.assertIsInstance(xml_string, bytes)
def test_fromstring_with_nonascii_text(self): def test_fromstring_with_nonascii_text(self):
input_str = '<?xml version="1.0" encoding="UTF-8"?>\n' \ input_str = b'<?xml version="1.0" encoding="UTF-8"?>\n' \
'<Test><FOO>\xef\xbc\xa1</FOO></Test>' b'<Test><FOO>\xef\xbc\xa1</FOO></Test>'
elem = etree.fromstring(input_str) elem = etree.fromstring(input_str)
text = elem.find('FOO').text text = elem.find('FOO').text
self.assertEqual(text, '\xef\xbc\xa1') if six.PY2:
self.assertTrue(isinstance(text, str)) self.assertEqual(text, b'\xef\xbc\xa1')
else:
self.assertEqual(text, b'\xef\xbc\xa1'.decode('utf8'))
self.assertIsInstance(text, str)
if __name__ == '__main__': if __name__ == '__main__':

29
test/unit/common/middleware/s3api/test_multi_delete.py
View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import base64
import json import json
import unittest import unittest
from datetime import datetime from datetime import datetime
@ -43,7 +44,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
obj = SubElement(elem, 'Object') obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = 'object' SubElement(obj, 'Key').text = 'object'
body = tostring(elem, use_s3ns=False) body = tostring(elem, use_s3ns=False)
content_md5 = md5(body).digest().encode('base64').strip() content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket/object?delete', req = Request.blank('/bucket/object?delete',
environ={'REQUEST_METHOD': 'POST'}, environ={'REQUEST_METHOD': 'POST'},
@ -80,7 +81,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
obj = SubElement(elem, 'Object') obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key SubElement(obj, 'Key').text = key
body = tostring(elem, use_s3ns=False) body = tostring(elem, use_s3ns=False)
content_md5 = md5(body).digest().encode('base64').strip() content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete', req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'}, environ={'REQUEST_METHOD': 'POST'},
@ -133,7 +134,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
obj = SubElement(elem, 'Object') obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key SubElement(obj, 'Key').text = key
body = tostring(elem, use_s3ns=False) body = tostring(elem, use_s3ns=False)
content_md5 = md5(body).digest().encode('base64').strip() content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete', req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'}, environ={'REQUEST_METHOD': 'POST'},
@ -180,7 +181,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
obj = SubElement(elem, 'Object') obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key SubElement(obj, 'Key').text = key
body = tostring(elem, use_s3ns=False) body = tostring(elem, use_s3ns=False)
content_md5 = md5(body).digest().encode('base64').strip() content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete', req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'}, environ={'REQUEST_METHOD': 'POST'},
@ -207,7 +208,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
obj = SubElement(elem, 'Object') obj = SubElement(elem, 'Object')
SubElement(obj, 'Key') SubElement(obj, 'Key')
body = tostring(elem, use_s3ns=False) body = tostring(elem, use_s3ns=False)
content_md5 = md5(body).digest().encode('base64').strip() content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete', req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'}, environ={'REQUEST_METHOD': 'POST'},
@ -232,7 +233,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
SubElement(obj, 'Key').text = key SubElement(obj, 'Key').text = key
SubElement(obj, 'VersionId').text = 'not-supported' SubElement(obj, 'VersionId').text = 'not-supported'
body = tostring(elem, use_s3ns=False) body = tostring(elem, use_s3ns=False)
content_md5 = md5(body).digest().encode('base64').strip() content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete', req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'}, environ={'REQUEST_METHOD': 'POST'},
@ -286,7 +287,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
obj = SubElement(elem, 'Object') obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = name SubElement(obj, 'Key').text = name
body = tostring(elem, use_s3ns=False) body = tostring(elem, use_s3ns=False)
content_md5 = md5(body).digest().encode('base64').strip() content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete', req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'}, environ={'REQUEST_METHOD': 'POST'},
@ -308,7 +309,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
obj = SubElement(elem, 'Object') obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = 'x' * 1000 + str(i) SubElement(obj, 'Key').text = 'x' * 1000 + str(i)
body = tostring(elem, use_s3ns=False) body = tostring(elem, use_s3ns=False)
content_md5 = md5(body).digest().encode('base64').strip() content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete', req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'}, environ={'REQUEST_METHOD': 'POST'},
@ -333,7 +334,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
obj = SubElement(elem, 'Object') obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key SubElement(obj, 'Key').text = key
body = tostring(elem, use_s3ns=False) body = tostring(elem, use_s3ns=False)
content_md5 = md5(body).digest().encode('base64').strip() content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete', req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'}, environ={'REQUEST_METHOD': 'POST'},
@ -374,8 +375,8 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
self.assertEqual(len(elem.findall('Deleted')), len(self.keys)) self.assertEqual(len(elem.findall('Deleted')), len(self.keys))
def _test_no_body(self, use_content_length=False, def _test_no_body(self, use_content_length=False,
use_transfer_encoding=False, string_to_md5=''): use_transfer_encoding=False, string_to_md5=b''):
content_md5 = md5(string_to_md5).digest().encode('base64').strip() content_md5 = base64.b64encode(md5(string_to_md5).digest()).strip()
with UnreadableInput(self) as fake_input: with UnreadableInput(self) as fake_input:
req = Request.blank( req = Request.blank(
'/bucket?delete', '/bucket?delete',
@ -398,11 +399,11 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
@s3acl @s3acl
def test_object_multi_DELETE_empty_body(self): def test_object_multi_DELETE_empty_body(self):
self._test_no_body() self._test_no_body()
self._test_no_body(string_to_md5='test') self._test_no_body(string_to_md5=b'test')
self._test_no_body(use_content_length=True) self._test_no_body(use_content_length=True)
self._test_no_body(use_content_length=True, string_to_md5='test') self._test_no_body(use_content_length=True, string_to_md5=b'test')
self._test_no_body(use_transfer_encoding=True) self._test_no_body(use_transfer_encoding=True)
self._test_no_body(use_transfer_encoding=True, string_to_md5='test') self._test_no_body(use_transfer_encoding=True, string_to_md5=b'test')
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

73
test/unit/common/middleware/s3api/test_multi_upload.py
View File

@ -14,12 +14,13 @@
# limitations under the License. # limitations under the License.
import base64 import base64
import binascii
import hashlib import hashlib
from mock import patch from mock import patch
import os import os
import time import time
import unittest import unittest
from urllib import quote from six.moves.urllib.parse import quote
from swift.common import swob from swift.common import swob
from swift.common.swob import Request from swift.common.swob import Request
@ -66,9 +67,9 @@ MULTIPARTS_TEMPLATE = \
('subdir/object/Z/2', '2014-05-07T19:47:58.592270', 'fedcba9876543210', ('subdir/object/Z/2', '2014-05-07T19:47:58.592270', 'fedcba9876543210',
41)) 41))
S3_ETAG = '"%s-2"' % hashlib.md5(( S3_ETAG = '"%s-2"' % hashlib.md5(binascii.a2b_hex(
'0123456789abcdef0123456789abcdef' '0123456789abcdef0123456789abcdef'
'fedcba9876543210fedcba9876543210').decode('hex')).hexdigest() 'fedcba9876543210fedcba9876543210')).hexdigest()
class TestS3ApiMultiUpload(S3ApiTestCase): class TestS3ApiMultiUpload(S3ApiTestCase):
@ -83,9 +84,9 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
self.s3api.conf.min_segment_size = 1 self.s3api.conf.min_segment_size = 1
objects = map(lambda item: {'name': item[0], 'last_modified': item[1], objects = [{'name': item[0], 'last_modified': item[1],
'hash': item[2], 'bytes': item[3]}, 'hash': item[2], 'bytes': item[3]}
OBJECTS_TEMPLATE) for item in OBJECTS_TEMPLATE]
object_list = json.dumps(objects) object_list = json.dumps(objects)
self.swift.register('PUT', segment_bucket, self.swift.register('PUT', segment_bucket,
@ -172,10 +173,10 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
multiparts=None): multiparts=None):
segment_bucket = '/v1/AUTH_test/bucket+segments' segment_bucket = '/v1/AUTH_test/bucket+segments'
objects = multiparts or MULTIPARTS_TEMPLATE objects = multiparts or MULTIPARTS_TEMPLATE
objects = map(lambda item: {'name': item[0], 'last_modified': item[1], objects = [{'name': item[0], 'last_modified': item[1],
'hash': item[2], 'bytes': item[3]}, 'hash': item[2], 'bytes': item[3]}
objects) for item in objects]
object_list = json.dumps(objects) object_list = json.dumps(objects).encode('ascii')
self.swift.register('GET', segment_bucket, swob.HTTPOk, {}, self.swift.register('GET', segment_bucket, swob.HTTPOk, {},
object_list) object_list)
@ -568,7 +569,7 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
self._test_object_multipart_upload_initiate({}) self._test_object_multipart_upload_initiate({})
self._test_object_multipart_upload_initiate({'Etag': 'blahblahblah'}) self._test_object_multipart_upload_initiate({'Etag': 'blahblahblah'})
self._test_object_multipart_upload_initiate({ self._test_object_multipart_upload_initiate({
'Content-MD5': base64.b64encode('blahblahblahblah').strip()}) 'Content-MD5': base64.b64encode(b'blahblahblahblah').strip()})
@s3acl(s3acl_only=True) @s3acl(s3acl_only=True)
@patch('swift.common.middleware.s3api.controllers.multi_upload.' @patch('swift.common.middleware.s3api.controllers.multi_upload.'
@ -667,7 +668,8 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
self.assertEqual(self._get_error_code(body), 'NoSuchBucket') self.assertEqual(self._get_error_code(body), 'NoSuchBucket')
def test_object_multipart_upload_complete(self): def test_object_multipart_upload_complete(self):
content_md5 = base64.b64encode(hashlib.md5(XML).digest()) content_md5 = base64.b64encode(hashlib.md5(
XML.encode('ascii')).digest())
req = Request.blank('/bucket/object?uploadId=X', req = Request.blank('/bucket/object?uploadId=X',
environ={'REQUEST_METHOD': 'POST'}, environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization': 'AWS test:tester:hmac', headers={'Authorization': 'AWS test:tester:hmac',
@ -701,7 +703,8 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
self.assertEqual(headers.get(h), override_etag) self.assertEqual(headers.get(h), override_etag)
def test_object_multipart_upload_invalid_md5(self): def test_object_multipart_upload_invalid_md5(self):
bad_md5 = base64.b64encode(hashlib.md5(XML + 'some junk').digest()) bad_md5 = base64.b64encode(hashlib.md5(
XML.encode('ascii') + b'some junk').digest())
req = Request.blank('/bucket/object?uploadId=X', req = Request.blank('/bucket/object?uploadId=X',
environ={'REQUEST_METHOD': 'POST'}, environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization': 'AWS test:tester:hmac', headers={'Authorization': 'AWS test:tester:hmac',
@ -726,11 +729,11 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
])) ]))
self.swift.register( self.swift.register(
'PUT', '/v1/AUTH_test/bucket/heartbeat-ok', 'PUT', '/v1/AUTH_test/bucket/heartbeat-ok',
swob.HTTPAccepted, {}, [' ', ' ', ' ', json.dumps({ swob.HTTPAccepted, {}, [b' ', b' ', b' ', json.dumps({
'Etag': '"slo-etag"', 'Etag': '"slo-etag"',
'Response Status': '201 Created', 'Response Status': '201 Created',
'Errors': [], 'Errors': [],
})]) }).encode('ascii')])
mock_time.time.side_effect = ( mock_time.time.side_effect = (
1, # start_time 1, # start_time
12, # first whitespace 12, # first whitespace
@ -748,14 +751,14 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
'Date': self.get_date_header(), }, 'Date': self.get_date_header(), },
body=XML) body=XML)
status, headers, body = self.call_s3api(req) status, headers, body = self.call_s3api(req)
lines = body.split('\n') lines = body.split(b'\n')
self.assertTrue(lines[0].startswith('<?xml ')) self.assertTrue(lines[0].startswith(b'<?xml '))
self.assertTrue(lines[1]) self.assertTrue(lines[1])
self.assertFalse(lines[1].strip()) self.assertFalse(lines[1].strip())
fromstring(body, 'CompleteMultipartUploadResult') fromstring(body, 'CompleteMultipartUploadResult')
self.assertEqual(status.split()[0], '200') self.assertEqual(status.split()[0], '200')
# NB: S3_ETAG includes quotes # NB: S3_ETAG includes quotes
self.assertIn('<ETag>%s</ETag>' % S3_ETAG, body) self.assertIn(('<ETag>%s</ETag>' % S3_ETAG).encode('ascii'), body)
self.assertEqual(self.swift.calls, [ self.assertEqual(self.swift.calls, [
('HEAD', '/v1/AUTH_test/bucket'), ('HEAD', '/v1/AUTH_test/bucket'),
('HEAD', '/v1/AUTH_test/bucket+segments/heartbeat-ok/X'), ('HEAD', '/v1/AUTH_test/bucket+segments/heartbeat-ok/X'),
@ -779,10 +782,10 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
])) ]))
self.swift.register( self.swift.register(
'PUT', '/v1/AUTH_test/bucket/heartbeat-fail', 'PUT', '/v1/AUTH_test/bucket/heartbeat-fail',
swob.HTTPAccepted, {}, [' ', ' ', ' ', json.dumps({ swob.HTTPAccepted, {}, [b' ', b' ', b' ', json.dumps({
'Response Status': '400 Bad Request', 'Response Status': '400 Bad Request',
'Errors': [['some/object', '403 Forbidden']], 'Errors': [['some/object', '403 Forbidden']],
})]) }).encode('ascii')])
mock_time.time.side_effect = ( mock_time.time.side_effect = (
1, # start_time 1, # start_time
12, # first whitespace 12, # first whitespace
@ -797,8 +800,8 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
'Date': self.get_date_header(), }, 'Date': self.get_date_header(), },
body=XML) body=XML)
status, headers, body = self.call_s3api(req) status, headers, body = self.call_s3api(req)
lines = body.split('\n') lines = body.split(b'\n')
self.assertTrue(lines[0].startswith('<?xml '), (status, lines)) self.assertTrue(lines[0].startswith(b'<?xml '), (status, lines))
self.assertTrue(lines[1]) self.assertTrue(lines[1])
self.assertFalse(lines[1].strip()) self.assertFalse(lines[1].strip())
fromstring(body, 'Error') fromstring(body, 'Error')
@ -828,10 +831,10 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
])) ]))
self.swift.register( self.swift.register(
'PUT', '/v1/AUTH_test/bucket/heartbeat-fail', 'PUT', '/v1/AUTH_test/bucket/heartbeat-fail',
swob.HTTPAccepted, {}, [' ', ' ', ' ', json.dumps({ swob.HTTPAccepted, {}, [b' ', b' ', b' ', json.dumps({
'Response Status': '400 Bad Request', 'Response Status': '400 Bad Request',
'Errors': [['some/object', '404 Not Found']], 'Errors': [['some/object', '404 Not Found']],
})]) }).encode('ascii')])
mock_time.time.side_effect = ( mock_time.time.side_effect = (
1, # start_time 1, # start_time
12, # first whitespace 12, # first whitespace
@ -846,8 +849,8 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
'Date': self.get_date_header(), }, 'Date': self.get_date_header(), },
body=XML) body=XML)
status, headers, body = self.call_s3api(req) status, headers, body = self.call_s3api(req)
lines = body.split('\n') lines = body.split(b'\n')
self.assertTrue(lines[0].startswith('<?xml ')) self.assertTrue(lines[0].startswith(b'<?xml '))
self.assertTrue(lines[1]) self.assertTrue(lines[1])
self.assertFalse(lines[1].strip()) self.assertFalse(lines[1].strip())
fromstring(body, 'Error') fromstring(body, 'Error')
@ -1116,8 +1119,8 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
self.assertEqual(status.split()[0], '200') self.assertEqual(status.split()[0], '200')
elem = fromstring(body, 'CompleteMultipartUploadResult') elem = fromstring(body, 'CompleteMultipartUploadResult')
self.assertNotIn('Etag', headers) self.assertNotIn('Etag', headers)
expected_etag = '"%s-3"' % hashlib.md5(''.join( expected_etag = '"%s-3"' % hashlib.md5(binascii.unhexlify(''.join(
x['hash'] for x in object_list).decode('hex')).hexdigest() x['hash'] for x in object_list))).hexdigest()
self.assertEqual(elem.find('ETag').text, expected_etag) self.assertEqual(elem.find('ETag').text, expected_etag)
self.assertEqual(self.swift.calls, [ self.assertEqual(self.swift.calls, [
@ -1843,8 +1846,8 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
account, src_headers={'Content-Length': '10'}, put_header=header) account, src_headers={'Content-Length': '10'}, put_header=header)
self.assertEqual(status.split()[0], '400') self.assertEqual(status.split()[0], '400')
self.assertIn('Range specified is not valid for ' self.assertIn(b'Range specified is not valid for '
'source object of size: 10', body) b'source object of size: 10', body)
self.assertEqual([ self.assertEqual([
('HEAD', '/v1/AUTH_test/bucket'), ('HEAD', '/v1/AUTH_test/bucket'),
@ -1887,9 +1890,9 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
self.assertEqual('/src_bucket/src_obj', put_headers['X-Copy-From']) self.assertEqual('/src_bucket/src_obj', put_headers['X-Copy-From'])
def _test_no_body(self, use_content_length=False, def _test_no_body(self, use_content_length=False,
use_transfer_encoding=False, string_to_md5=''): use_transfer_encoding=False, string_to_md5=b''):
raw_md5 = hashlib.md5(string_to_md5).digest() raw_md5 = hashlib.md5(string_to_md5).digest()
content_md5 = raw_md5.encode('base64').strip() content_md5 = base64.b64encode(raw_md5).strip()
with UnreadableInput(self) as fake_input: with UnreadableInput(self) as fake_input:
req = Request.blank( req = Request.blank(
'/bucket/object?uploadId=X', '/bucket/object?uploadId=X',
@ -1914,11 +1917,11 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
@s3acl @s3acl
def test_object_multi_upload_empty_body(self): def test_object_multi_upload_empty_body(self):
self._test_no_body() self._test_no_body()
self._test_no_body(string_to_md5='test') self._test_no_body(string_to_md5=b'test')
self._test_no_body(use_content_length=True) self._test_no_body(use_content_length=True)
self._test_no_body(use_content_length=True, string_to_md5='test') self._test_no_body(use_content_length=True, string_to_md5=b'test')
self._test_no_body(use_transfer_encoding=True) self._test_no_body(use_transfer_encoding=True)
self._test_no_body(use_transfer_encoding=True, string_to_md5='test') self._test_no_body(use_transfer_encoding=True, string_to_md5=b'test')
class TestS3ApiMultiUploadNonUTC(TestS3ApiMultiUpload): class TestS3ApiMultiUploadNonUTC(TestS3ApiMultiUpload):

30
test/unit/common/middleware/s3api/test_obj.py
View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import binascii
import unittest import unittest
from datetime import datetime from datetime import datetime
import hashlib import hashlib
@ -20,6 +21,7 @@ import os
from os.path import join from os.path import join
import time import time
from mock import patch from mock import patch
import six
from swift.common import swob from swift.common import swob
from swift.common.swob import Request from swift.common.swob import Request
@ -37,7 +39,7 @@ class TestS3ApiObj(S3ApiTestCase):
def setUp(self): def setUp(self):
super(TestS3ApiObj, self).setUp() super(TestS3ApiObj, self).setUp()
self.object_body = 'hello' self.object_body = b'hello'
self.etag = hashlib.md5(self.object_body).hexdigest() self.etag = hashlib.md5(self.object_body).hexdigest()
self.last_modified = 'Fri, 01 Apr 2014 12:00:00 GMT' self.last_modified = 'Fri, 01 Apr 2014 12:00:00 GMT'
@ -110,7 +112,7 @@ class TestS3ApiObj(S3ApiTestCase):
swob.HTTPUnauthorized, {}, None) swob.HTTPUnauthorized, {}, None)
status, headers, body = self.call_s3api(req) status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '403') self.assertEqual(status.split()[0], '403')
self.assertEqual(body, '') # sanity self.assertEqual(body, b'') # sanity
req = Request.blank('/bucket/object', req = Request.blank('/bucket/object',
environ={'REQUEST_METHOD': 'HEAD'}, environ={'REQUEST_METHOD': 'HEAD'},
@ -120,7 +122,7 @@ class TestS3ApiObj(S3ApiTestCase):
swob.HTTPForbidden, {}, None) swob.HTTPForbidden, {}, None)
status, headers, body = self.call_s3api(req) status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '403') self.assertEqual(status.split()[0], '403')
self.assertEqual(body, '') # sanity self.assertEqual(body, b'') # sanity
req = Request.blank('/bucket/object', req = Request.blank('/bucket/object',
environ={'REQUEST_METHOD': 'HEAD'}, environ={'REQUEST_METHOD': 'HEAD'},
@ -130,7 +132,7 @@ class TestS3ApiObj(S3ApiTestCase):
swob.HTTPNotFound, {}, None) swob.HTTPNotFound, {}, None)
status, headers, body = self.call_s3api(req) status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '404') self.assertEqual(status.split()[0], '404')
self.assertEqual(body, '') # sanity self.assertEqual(body, b'') # sanity
req = Request.blank('/bucket/object', req = Request.blank('/bucket/object',
environ={'REQUEST_METHOD': 'HEAD'}, environ={'REQUEST_METHOD': 'HEAD'},
@ -140,7 +142,7 @@ class TestS3ApiObj(S3ApiTestCase):
swob.HTTPPreconditionFailed, {}, None) swob.HTTPPreconditionFailed, {}, None)
status, headers, body = self.call_s3api(req) status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '412') self.assertEqual(status.split()[0], '412')
self.assertEqual(body, '') # sanity self.assertEqual(body, b'') # sanity
req = Request.blank('/bucket/object', req = Request.blank('/bucket/object',
environ={'REQUEST_METHOD': 'HEAD'}, environ={'REQUEST_METHOD': 'HEAD'},
@ -150,7 +152,7 @@ class TestS3ApiObj(S3ApiTestCase):
swob.HTTPServerError, {}, None) swob.HTTPServerError, {}, None)
status, headers, body = self.call_s3api(req) status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '500') self.assertEqual(status.split()[0], '500')
self.assertEqual(body, '') # sanity self.assertEqual(body, b'') # sanity
req = Request.blank('/bucket/object', req = Request.blank('/bucket/object',
environ={'REQUEST_METHOD': 'HEAD'}, environ={'REQUEST_METHOD': 'HEAD'},
@ -160,7 +162,7 @@ class TestS3ApiObj(S3ApiTestCase):
swob.HTTPServiceUnavailable, {}, None) swob.HTTPServiceUnavailable, {}, None)
status, headers, body = self.call_s3api(req) status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '500') self.assertEqual(status.split()[0], '500')
self.assertEqual(body, '') # sanity self.assertEqual(body, b'') # sanity
def test_object_HEAD(self): def test_object_HEAD(self):
self._test_object_GETorHEAD('HEAD') self._test_object_GETorHEAD('HEAD')
@ -448,7 +450,9 @@ class TestS3ApiObj(S3ApiTestCase):
@s3acl @s3acl
def test_object_PUT(self): def test_object_PUT(self):
etag = self.response_headers['etag'] etag = self.response_headers['etag']
content_md5 = etag.decode('hex').encode('base64').strip() content_md5 = binascii.b2a_base64(binascii.a2b_hex(etag)).strip()
if not six.PY2:
content_md5 = content_md5.decode('ascii')
req = Request.blank( req = Request.blank(
'/bucket/object', '/bucket/object',
@ -524,7 +528,9 @@ class TestS3ApiObj(S3ApiTestCase):
self.assertEqual(self._get_error_code(body), 'BadDigest') self.assertEqual(self._get_error_code(body), 'BadDigest')
def test_object_PUT_headers(self): def test_object_PUT_headers(self):
content_md5 = self.etag.decode('hex').encode('base64').strip() content_md5 = binascii.b2a_base64(binascii.a2b_hex(self.etag)).strip()
if not six.PY2:
content_md5 = content_md5.decode('ascii')
self.swift.register('HEAD', '/v1/AUTH_test/some/source', self.swift.register('HEAD', '/v1/AUTH_test/some/source',
swob.HTTPOk, {'last-modified': self.last_modified}, swob.HTTPOk, {'last-modified': self.last_modified},
@ -540,10 +546,12 @@ class TestS3ApiObj(S3ApiTestCase):
'X-Amz-Meta-Lots-Of-Unprintable': 5 * '\x04', 'X-Amz-Meta-Lots-Of-Unprintable': 5 * '\x04',
'X-Amz-Copy-Source': '/some/source', 'X-Amz-Copy-Source': '/some/source',
'Content-MD5': content_md5, 'Content-MD5': content_md5,
'Date': self.get_date_header()}) 'Date': self.get_date_header()},
body=self.object_body)
req.date = datetime.now() req.date = datetime.now()
req.content_type = 'text/plain' req.content_type = 'text/plain'
status, headers, body = self.call_s3api(req) status, headers, body = self.call_s3api(req)
self.assertEqual('200 ', status[:4], body)
# Check that s3api does not return an etag header, # Check that s3api does not return an etag header,
# specified copy source. # specified copy source.
self.assertTrue(headers.get('etag') is None) self.assertTrue(headers.get('etag') is None)
@ -1002,7 +1010,7 @@ class TestS3ApiObj(S3ApiTestCase):
'Content-Type': 'foo/bar'}) 'Content-Type': 'foo/bar'})
status, headers, body = self.call_s3api(req) status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '204') self.assertEqual(status.split()[0], '204')
self.assertEqual(body, '') self.assertEqual(body, b'')
self.assertIn(('HEAD', '/v1/AUTH_test/bucket/object'), self.assertIn(('HEAD', '/v1/AUTH_test/bucket/object'),
self.swift.calls) self.swift.calls)

9
test/unit/common/middleware/s3api/test_s3_acl.py
View File

@ -66,9 +66,10 @@ def s3acl(func=None, s3acl_only=False):
exc_type, exc_instance, exc_traceback = sys.exc_info() exc_type, exc_instance, exc_traceback = sys.exc_info()
formatted_traceback = ''.join(traceback.format_tb( formatted_traceback = ''.join(traceback.format_tb(
exc_traceback)) exc_traceback))
message = '\n%s\n%s:\n%s' % (formatted_traceback, message = '\n%s\n%s' % (formatted_traceback,
exc_type.__name__, exc_type.__name__)
exc_instance.message) if exc_instance.args:
message += ':\n%s' % (exc_instance.args[0],)
message += failing_point message += failing_point
raise exc_type(message) raise exc_type(message)
@ -114,7 +115,7 @@ def generate_s3acl_environ(account, swift, owner):
account_name = '%s:%s' % (account, permission.lower()) account_name = '%s:%s' % (account, permission.lower())
return Grant(User(account_name), permission) return Grant(User(account_name), permission)
grants = map(gen_grant, PERMISSIONS) grants = [gen_grant(perm) for perm in PERMISSIONS]
container_headers = _gen_test_headers(owner, grants) container_headers = _gen_test_headers(owner, grants)
object_headers = _gen_test_headers(owner, grants, 'object') object_headers = _gen_test_headers(owner, grants, 'object')
object_body = 'hello' object_body = 'hello'

58
test/unit/common/middleware/s3api/test_s3api.py
View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import base64
import unittest import unittest
from mock import patch, MagicMock from mock import patch, MagicMock
import calendar import calendar
@ -22,7 +23,8 @@ import mock
import requests import requests
import json import json
import copy import copy
from urllib import unquote, quote import six
from six.moves.urllib.parse import unquote, quote
import swift.common.middleware.s3api import swift.common.middleware.s3api
from swift.common.middleware.keystoneauth import KeystoneAuth from swift.common.middleware.keystoneauth import KeystoneAuth
@ -100,7 +102,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
def test_non_s3_request_passthrough(self): def test_non_s3_request_passthrough(self):
req = Request.blank('/something') req = Request.blank('/something')
status, headers, body = self.call_s3api(req) status, headers, body = self.call_s3api(req)
self.assertEqual(body, 'FAKE APP') self.assertEqual(body, b'FAKE APP')
def test_bad_format_authorization(self): def test_bad_format_authorization(self):
req = Request.blank('/something', req = Request.blank('/something',
@ -336,7 +338,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
self.assertIsNone(headers['X-Auth-Token']) self.assertIsNone(headers['X-Auth-Token'])
def test_signed_urls_v4_bad_credential(self): def test_signed_urls_v4_bad_credential(self):
def test(credential, message, extra=''): def test(credential, message, extra=b''):
req = Request.blank( req = Request.blank(
'/bucket/object' '/bucket/object'
'?X-Amz-Algorithm=AWS4-HMAC-SHA256' '?X-Amz-Algorithm=AWS4-HMAC-SHA256'
@ -364,7 +366,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
test('test:tester/%s/us-west-1/s3/aws4_request' % dt, test('test:tester/%s/us-west-1/s3/aws4_request' % dt,
"Error parsing the X-Amz-Credential parameter; the region " "Error parsing the X-Amz-Credential parameter; the region "
"'us-west-1' is wrong; expecting 'us-east-1'", "'us-west-1' is wrong; expecting 'us-east-1'",
'<Region>us-east-1</Region>') b'<Region>us-east-1</Region>')
test('test:tester/%s/us-east-1/not-s3/aws4_request' % dt, test('test:tester/%s/us-east-1/not-s3/aws4_request' % dt,
'Error parsing the X-Amz-Credential parameter; incorrect service ' 'Error parsing the X-Amz-Credential parameter; incorrect service '
'"not-s3". This endpoint belongs to "s3".') '"not-s3". This endpoint belongs to "s3".')
@ -483,9 +485,9 @@ class TestS3ApiMiddleware(S3ApiTestCase):
self.assertEqual(req.environ['s3api.auth_details'], { self.assertEqual(req.environ['s3api.auth_details'], {
'access_key': 'test:tester', 'access_key': 'test:tester',
'signature': 'hmac', 'signature': 'hmac',
'string_to_sign': '\n'.join([ 'string_to_sign': b'\n'.join([
'PUT', '', '', date_header, b'PUT', b'', b'', date_header.encode('ascii'),
'/bucket/object?partNumber=1&uploadId=123456789abcdef']), b'/bucket/object?partNumber=1&uploadId=123456789abcdef']),
'check_signature': mock_cs}) 'check_signature': mock_cs})
def test_invalid_uri(self): def test_invalid_uri(self):
@ -506,8 +508,10 @@ class TestS3ApiMiddleware(S3ApiTestCase):
self.assertEqual(self._get_error_code(body), 'InvalidDigest') self.assertEqual(self._get_error_code(body), 'InvalidDigest')
def test_object_create_bad_md5_too_short(self): def test_object_create_bad_md5_too_short(self):
too_short_digest = hashlib.md5('hey').hexdigest()[:-1] too_short_digest = hashlib.md5(b'hey').digest()[:-1]
md5_str = too_short_digest.encode('base64').strip() md5_str = base64.b64encode(too_short_digest).strip()
if not six.PY2:
md5_str = md5_str.decode('ascii')
req = Request.blank( req = Request.blank(
'/bucket/object', '/bucket/object',
environ={'REQUEST_METHOD': 'PUT', environ={'REQUEST_METHOD': 'PUT',
@ -518,8 +522,10 @@ class TestS3ApiMiddleware(S3ApiTestCase):
self.assertEqual(self._get_error_code(body), 'InvalidDigest') self.assertEqual(self._get_error_code(body), 'InvalidDigest')
def test_object_create_bad_md5_too_long(self): def test_object_create_bad_md5_too_long(self):
too_long_digest = hashlib.md5('hey').hexdigest() + 'suffix' too_long_digest = hashlib.md5(b'hey').digest() + b'suffix'
md5_str = too_long_digest.encode('base64').strip() md5_str = base64.b64encode(too_long_digest).strip()
if not six.PY2:
md5_str = md5_str.decode('ascii')
req = Request.blank( req = Request.blank(
'/bucket/object', '/bucket/object',
environ={'REQUEST_METHOD': 'PUT', environ={'REQUEST_METHOD': 'PUT',
@ -705,13 +711,13 @@ class TestS3ApiMiddleware(S3ApiTestCase):
with self.assertRaises(ValueError) as cm: with self.assertRaises(ValueError) as cm:
self.s3api.check_pipeline(self.conf) self.s3api.check_pipeline(self.conf)
self.assertIn('expected auth between s3api and proxy-server', self.assertIn('expected auth between s3api and proxy-server',
cm.exception.message) cm.exception.args[0])
pipeline.return_value = 'proxy-server' pipeline.return_value = 'proxy-server'
with self.assertRaises(ValueError) as cm: with self.assertRaises(ValueError) as cm:
self.s3api.check_pipeline(self.conf) self.s3api.check_pipeline(self.conf)
self.assertIn("missing filters ['s3api']", self.assertIn("missing filters ['s3api']",
cm.exception.message) cm.exception.args[0])
def test_s3api_initialization_with_disabled_pipeline_check(self): def test_s3api_initialization_with_disabled_pipeline_check(self):
with patch("swift.common.middleware.s3api.s3api.loadcontext"), \ with patch("swift.common.middleware.s3api.s3api.loadcontext"), \
@ -799,7 +805,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
'Missing required header for this request: x-amz-content-sha256') 'Missing required header for this request: x-amz-content-sha256')
def test_signature_v4_bad_authorization_string(self): def test_signature_v4_bad_authorization_string(self):
def test(auth_str, error, msg, extra=''): def test(auth_str, error, msg, extra=b''):
environ = { environ = {
'REQUEST_METHOD': 'GET'} 'REQUEST_METHOD': 'GET'}
headers = { headers = {
@ -835,7 +841,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
test(auth_str, 'AuthorizationHeaderMalformed', test(auth_str, 'AuthorizationHeaderMalformed',
"The authorization header is malformed; " "The authorization header is malformed; "
"the region 'us-west-2' is wrong; expecting 'us-east-1'", "the region 'us-west-2' is wrong; expecting 'us-east-1'",
'<Region>us-east-1</Region>') b'<Region>us-east-1</Region>')
auth_str = ('AWS4-HMAC-SHA256 ' auth_str = ('AWS4-HMAC-SHA256 '
'Credential=test:tester/%s/us-east-1/not-s3/aws4_request, ' 'Credential=test:tester/%s/us-east-1/not-s3/aws4_request, '
@ -901,8 +907,8 @@ class TestS3ApiMiddleware(S3ApiTestCase):
patch.object(swift.common.middleware.s3api.s3request, patch.object(swift.common.middleware.s3api.s3request,
'SERVICE', 'host'): 'SERVICE', 'host'):
req = _get_req(path, environ) req = _get_req(path, environ)
hash_in_sts = req._string_to_sign().split('\n')[3] hash_in_sts = req._string_to_sign().split(b'\n')[3]
self.assertEqual(hash_val, hash_in_sts) self.assertEqual(hash_val, hash_in_sts.decode('ascii'))
self.assertTrue(req.check_signature( self.assertTrue(req.check_signature(
'wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY')) 'wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY'))
@ -963,7 +969,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
'validate_bucket_name'): 'validate_bucket_name'):
verify('27ba31df5dbc6e063d8f87d62eb07143' verify('27ba31df5dbc6e063d8f87d62eb07143'
'f7f271c5330a917840586ac1c85b6f6b', 'f7f271c5330a917840586ac1c85b6f6b',
unquote('/%E1%88%B4'), env) swob.wsgi_unquote('/%E1%88%B4'), env)
# get-vanilla-query-order-key # get-vanilla-query-order-key
env = { env = {
@ -1101,12 +1107,12 @@ class TestS3ApiMiddleware(S3ApiTestCase):
swob.HTTPOk, {}, None) swob.HTTPOk, {}, None)
with patch.object(self.s3_token, '_json_request') as mock_req: with patch.object(self.s3_token, '_json_request') as mock_req:
mock_resp = requests.Response() mock_resp = requests.Response()
mock_resp._content = json.dumps(GOOD_RESPONSE_V2) mock_resp._content = json.dumps(GOOD_RESPONSE_V2).encode('ascii')
mock_resp.status_code = 201 mock_resp.status_code = 201
mock_req.return_value = mock_resp mock_req.return_value = mock_resp
status, headers, body = self.call_s3api(req) status, headers, body = self.call_s3api(req)
self.assertEqual(body, '') self.assertEqual(body, b'')
self.assertEqual(1, mock_req.call_count) self.assertEqual(1, mock_req.call_count)
def test_s3api_with_only_s3_token_v3(self): def test_s3api_with_only_s3_token_v3(self):
@ -1127,12 +1133,12 @@ class TestS3ApiMiddleware(S3ApiTestCase):
swob.HTTPOk, {}, None) swob.HTTPOk, {}, None)
with patch.object(self.s3_token, '_json_request') as mock_req: with patch.object(self.s3_token, '_json_request') as mock_req:
mock_resp = requests.Response() mock_resp = requests.Response()
mock_resp._content = json.dumps(GOOD_RESPONSE_V3) mock_resp._content = json.dumps(GOOD_RESPONSE_V3).encode('ascii')
mock_resp.status_code = 200 mock_resp.status_code = 200
mock_req.return_value = mock_resp mock_req.return_value = mock_resp
status, headers, body = self.call_s3api(req) status, headers, body = self.call_s3api(req)
self.assertEqual(body, '') self.assertEqual(body, b'')
self.assertEqual(1, mock_req.call_count) self.assertEqual(1, mock_req.call_count)
def test_s3api_with_s3_token_and_auth_token(self): def test_s3api_with_s3_token_and_auth_token(self):
@ -1157,7 +1163,8 @@ class TestS3ApiMiddleware(S3ApiTestCase):
with patch.object(self.auth_token, with patch.object(self.auth_token,
'_do_fetch_token') as mock_fetch: '_do_fetch_token') as mock_fetch:
mock_resp = requests.Response() mock_resp = requests.Response()
mock_resp._content = json.dumps(GOOD_RESPONSE_V2) mock_resp._content = json.dumps(
GOOD_RESPONSE_V2).encode('ascii')
mock_resp.status_code = 201 mock_resp.status_code = 201
mock_req.return_value = mock_resp mock_req.return_value = mock_resp
@ -1167,7 +1174,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
mock_fetch.return_value = (MagicMock(), mock_access_info) mock_fetch.return_value = (MagicMock(), mock_access_info)
status, headers, body = self.call_s3api(req) status, headers, body = self.call_s3api(req)
self.assertEqual(body, '') self.assertEqual(body, b'')
self.assertEqual(1, mock_req.call_count) self.assertEqual(1, mock_req.call_count)
# With X-Auth-Token, auth_token will call _do_fetch_token to # With X-Auth-Token, auth_token will call _do_fetch_token to
# connect to keystone in auth_token, again # connect to keystone in auth_token, again
@ -1198,7 +1205,8 @@ class TestS3ApiMiddleware(S3ApiTestCase):
no_token_id_good_resp = copy.deepcopy(GOOD_RESPONSE_V2) no_token_id_good_resp = copy.deepcopy(GOOD_RESPONSE_V2)
# delete token id # delete token id
del no_token_id_good_resp['access']['token']['id'] del no_token_id_good_resp['access']['token']['id']
mock_resp._content = json.dumps(no_token_id_good_resp) mock_resp._content = json.dumps(
no_token_id_good_resp).encode('ascii')
mock_resp.status_code = 201 mock_resp.status_code = 201
mock_req.return_value = mock_resp mock_req.return_value = mock_resp

106
test/unit/common/middleware/s3api/test_s3request.py
View File

@ -143,7 +143,7 @@ class TestRequest(S3ApiTestCase):
def test_get_response_without_match_ACL_MAP(self): def test_get_response_without_match_ACL_MAP(self):
with self.assertRaises(Exception) as e: with self.assertRaises(Exception) as e:
self._test_get_response('POST', req_klass=S3AclRequest) self._test_get_response('POST', req_klass=S3AclRequest)
self.assertEqual(e.exception.message, self.assertEqual(e.exception.args[0],
'No permission to be checked exists') 'No permission to be checked exists')
def test_get_response_without_duplication_HEAD_request(self): def test_get_response_without_duplication_HEAD_request(self):
@ -215,8 +215,8 @@ class TestRequest(S3ApiTestCase):
s3req = create_s3request_with_param('max-keys', '1' * 30) s3req = create_s3request_with_param('max-keys', '1' * 30)
with self.assertRaises(InvalidArgument) as result: with self.assertRaises(InvalidArgument) as result:
s3req.get_validated_param('max-keys', 1) s3req.get_validated_param('max-keys', 1)
self.assertTrue( self.assertIn(
'not an integer or within integer range' in result.exception.body) b'not an integer or within integer range', result.exception.body)
self.assertEqual( self.assertEqual(
result.exception.headers['content-type'], 'application/xml') result.exception.headers['content-type'], 'application/xml')
@ -224,8 +224,8 @@ class TestRequest(S3ApiTestCase):
s3req = create_s3request_with_param('max-keys', '-1') s3req = create_s3request_with_param('max-keys', '-1')
with self.assertRaises(InvalidArgument) as result: with self.assertRaises(InvalidArgument) as result:
s3req.get_validated_param('max-keys', 1) s3req.get_validated_param('max-keys', 1)
self.assertTrue( self.assertIn(
'must be an integer between 0 and' in result.exception.body) b'must be an integer between 0 and', result.exception.body)
self.assertEqual( self.assertEqual(
result.exception.headers['content-type'], 'application/xml') result.exception.headers['content-type'], 'application/xml')
@ -233,8 +233,8 @@ class TestRequest(S3ApiTestCase):
s3req = create_s3request_with_param('max-keys', 'invalid') s3req = create_s3request_with_param('max-keys', 'invalid')
with self.assertRaises(InvalidArgument) as result: with self.assertRaises(InvalidArgument) as result:
s3req.get_validated_param('max-keys', 1) s3req.get_validated_param('max-keys', 1)
self.assertTrue( self.assertIn(
'not an integer or within integer range' in result.exception.body) b'not an integer or within integer range', result.exception.body)
self.assertEqual( self.assertEqual(
result.exception.headers['content-type'], 'application/xml') result.exception.headers['content-type'], 'application/xml')
@ -351,7 +351,7 @@ class TestRequest(S3ApiTestCase):
headers={'Authorization': 'AWS test:tester:hmac'}) headers={'Authorization': 'AWS test:tester:hmac'})
status, headers, body = self.call_s3api(req) status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '403') self.assertEqual(status.split()[0], '403')
self.assertEqual(body, '') self.assertEqual(body, b'')
def test_date_header_expired(self): def test_date_header_expired(self):
self.swift.register('HEAD', '/v1/AUTH_test/nojunk', swob.HTTPNotFound, self.swift.register('HEAD', '/v1/AUTH_test/nojunk', swob.HTTPNotFound,
@ -363,7 +363,7 @@ class TestRequest(S3ApiTestCase):
status, headers, body = self.call_s3api(req) status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '403') self.assertEqual(status.split()[0], '403')
self.assertEqual(body, '') self.assertEqual(body, b'')
def test_date_header_with_x_amz_date_valid(self): def test_date_header_with_x_amz_date_valid(self):
self.swift.register('HEAD', '/v1/AUTH_test/nojunk', swob.HTTPNotFound, self.swift.register('HEAD', '/v1/AUTH_test/nojunk', swob.HTTPNotFound,
@ -376,7 +376,7 @@ class TestRequest(S3ApiTestCase):
status, headers, body = self.call_s3api(req) status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '404') self.assertEqual(status.split()[0], '404')
self.assertEqual(body, '') self.assertEqual(body, b'')
def test_date_header_with_x_amz_date_expired(self): def test_date_header_with_x_amz_date_expired(self):
self.swift.register('HEAD', '/v1/AUTH_test/nojunk', swob.HTTPNotFound, self.swift.register('HEAD', '/v1/AUTH_test/nojunk', swob.HTTPNotFound,
@ -390,7 +390,7 @@ class TestRequest(S3ApiTestCase):
status, headers, body = self.call_s3api(req) status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '403') self.assertEqual(status.split()[0], '403')
self.assertEqual(body, '') self.assertEqual(body, b'')
def _test_request_timestamp_sigv4(self, date_header): def _test_request_timestamp_sigv4(self, date_header):
# signature v4 here # signature v4 here
@ -428,7 +428,7 @@ class TestRequest(S3ApiTestCase):
def test_request_timestamp_sigv4(self): def test_request_timestamp_sigv4(self):
access_denied_message = \ access_denied_message = \
'AWS authentication requires a valid Date or x-amz-date header' b'AWS authentication requires a valid Date or x-amz-date header'
# normal X-Amz-Date header # normal X-Amz-Date header
date_header = {'X-Amz-Date': self.get_v4_amz_date_header()} date_header = {'X-Amz-Date': self.get_v4_amz_date_header()}
@ -443,7 +443,7 @@ class TestRequest(S3ApiTestCase):
with self.assertRaises(AccessDenied) as cm: with self.assertRaises(AccessDenied) as cm:
self._test_request_timestamp_sigv4(date_header) self._test_request_timestamp_sigv4(date_header)
self.assertEqual('403 Forbidden', cm.exception.message) self.assertEqual('403 Forbidden', cm.exception.args[0])
self.assertIn(access_denied_message, cm.exception.body) self.assertIn(access_denied_message, cm.exception.body)
# mangled Date header # mangled Date header
@ -451,7 +451,7 @@ class TestRequest(S3ApiTestCase):
with self.assertRaises(AccessDenied) as cm: with self.assertRaises(AccessDenied) as cm:
self._test_request_timestamp_sigv4(date_header) self._test_request_timestamp_sigv4(date_header)
self.assertEqual('403 Forbidden', cm.exception.message) self.assertEqual('403 Forbidden', cm.exception.args[0])
self.assertIn(access_denied_message, cm.exception.body) self.assertIn(access_denied_message, cm.exception.body)
# Negative timestamp # Negative timestamp
@ -459,7 +459,7 @@ class TestRequest(S3ApiTestCase):
with self.assertRaises(AccessDenied) as cm: with self.assertRaises(AccessDenied) as cm:
self._test_request_timestamp_sigv4(date_header) self._test_request_timestamp_sigv4(date_header)
self.assertEqual('403 Forbidden', cm.exception.message) self.assertEqual('403 Forbidden', cm.exception.args[0])
self.assertIn(access_denied_message, cm.exception.body) self.assertIn(access_denied_message, cm.exception.body)
# far-past Date header # far-past Date header
@ -467,7 +467,7 @@ class TestRequest(S3ApiTestCase):
with self.assertRaises(AccessDenied) as cm: with self.assertRaises(AccessDenied) as cm:
self._test_request_timestamp_sigv4(date_header) self._test_request_timestamp_sigv4(date_header)
self.assertEqual('403 Forbidden', cm.exception.message) self.assertEqual('403 Forbidden', cm.exception.args[0])
self.assertIn(access_denied_message, cm.exception.body) self.assertIn(access_denied_message, cm.exception.body)
# near-future X-Amz-Date header # near-future X-Amz-Date header
@ -481,9 +481,9 @@ class TestRequest(S3ApiTestCase):
with self.assertRaises(RequestTimeTooSkewed) as cm: with self.assertRaises(RequestTimeTooSkewed) as cm:
self._test_request_timestamp_sigv4(date_header) self._test_request_timestamp_sigv4(date_header)
self.assertEqual('403 Forbidden', cm.exception.message) self.assertEqual('403 Forbidden', cm.exception.args[0])
self.assertIn('The difference between the request time and the ' self.assertIn(b'The difference between the request time and the '
'current time is too large.', cm.exception.body) b'current time is too large.', cm.exception.body)
def _test_request_timestamp_sigv2(self, date_header): def _test_request_timestamp_sigv2(self, date_header):
# signature v4 here # signature v4 here
@ -505,7 +505,7 @@ class TestRequest(S3ApiTestCase):
def test_request_timestamp_sigv2(self): def test_request_timestamp_sigv2(self):
access_denied_message = \ access_denied_message = \
'AWS authentication requires a valid Date or x-amz-date header' b'AWS authentication requires a valid Date or x-amz-date header'
# In v2 format, normal X-Amz-Date header is same # In v2 format, normal X-Amz-Date header is same
date_header = {'X-Amz-Date': self.get_date_header()} date_header = {'X-Amz-Date': self.get_date_header()}
@ -520,7 +520,7 @@ class TestRequest(S3ApiTestCase):
with self.assertRaises(AccessDenied) as cm: with self.assertRaises(AccessDenied) as cm:
self._test_request_timestamp_sigv2(date_header) self._test_request_timestamp_sigv2(date_header)
self.assertEqual('403 Forbidden', cm.exception.message) self.assertEqual('403 Forbidden', cm.exception.args[0])
self.assertIn(access_denied_message, cm.exception.body) self.assertIn(access_denied_message, cm.exception.body)
# mangled Date header # mangled Date header
@ -528,7 +528,7 @@ class TestRequest(S3ApiTestCase):
with self.assertRaises(AccessDenied) as cm: with self.assertRaises(AccessDenied) as cm:
self._test_request_timestamp_sigv2(date_header) self._test_request_timestamp_sigv2(date_header)
self.assertEqual('403 Forbidden', cm.exception.message) self.assertEqual('403 Forbidden', cm.exception.args[0])
self.assertIn(access_denied_message, cm.exception.body) self.assertIn(access_denied_message, cm.exception.body)
# Negative timestamp # Negative timestamp
@ -536,7 +536,7 @@ class TestRequest(S3ApiTestCase):
with self.assertRaises(AccessDenied) as cm: with self.assertRaises(AccessDenied) as cm:
self._test_request_timestamp_sigv2(date_header) self._test_request_timestamp_sigv2(date_header)
self.assertEqual('403 Forbidden', cm.exception.message) self.assertEqual('403 Forbidden', cm.exception.args[0])
self.assertIn(access_denied_message, cm.exception.body) self.assertIn(access_denied_message, cm.exception.body)
# far-past Date header # far-past Date header
@ -544,7 +544,7 @@ class TestRequest(S3ApiTestCase):
with self.assertRaises(AccessDenied) as cm: with self.assertRaises(AccessDenied) as cm:
self._test_request_timestamp_sigv2(date_header) self._test_request_timestamp_sigv2(date_header)
self.assertEqual('403 Forbidden', cm.exception.message) self.assertEqual('403 Forbidden', cm.exception.args[0])
self.assertIn(access_denied_message, cm.exception.body) self.assertIn(access_denied_message, cm.exception.body)
# far-future Date header # far-future Date header
@ -552,9 +552,9 @@ class TestRequest(S3ApiTestCase):
with self.assertRaises(RequestTimeTooSkewed) as cm: with self.assertRaises(RequestTimeTooSkewed) as cm:
self._test_request_timestamp_sigv2(date_header) self._test_request_timestamp_sigv2(date_header)
self.assertEqual('403 Forbidden', cm.exception.message) self.assertEqual('403 Forbidden', cm.exception.args[0])
self.assertIn('The difference between the request time and the ' self.assertIn(b'The difference between the request time and the '
'current time is too large.', cm.exception.body) b'current time is too large.', cm.exception.body)
def test_headers_to_sign_sigv4(self): def test_headers_to_sign_sigv4(self):
environ = { environ = {
@ -681,14 +681,14 @@ class TestRequest(S3ApiTestCase):
sigv4_req = SigV4Request(req.environ) sigv4_req = SigV4Request(req.environ)
uri = sigv4_req._canonical_uri() uri = sigv4_req._canonical_uri()
self.assertEqual(uri, '/') self.assertEqual(uri, b'/')
self.assertEqual(req.environ['PATH_INFO'], '/') self.assertEqual(req.environ['PATH_INFO'], '/')
req = Request.blank('/obj1', environ=environ, headers=headers) req = Request.blank('/obj1', environ=environ, headers=headers)
sigv4_req = SigV4Request(req.environ) sigv4_req = SigV4Request(req.environ)
uri = sigv4_req._canonical_uri() uri = sigv4_req._canonical_uri()
self.assertEqual(uri, '/obj1') self.assertEqual(uri, b'/obj1')
self.assertEqual(req.environ['PATH_INFO'], '/obj1') self.assertEqual(req.environ['PATH_INFO'], '/obj1')
environ = { environ = {
@ -701,7 +701,7 @@ class TestRequest(S3ApiTestCase):
sigv4_req = SigV4Request(req.environ) sigv4_req = SigV4Request(req.environ)
uri = sigv4_req._canonical_uri() uri = sigv4_req._canonical_uri()
self.assertEqual(uri, '/') self.assertEqual(uri, b'/')
self.assertEqual(req.environ['PATH_INFO'], '/') self.assertEqual(req.environ['PATH_INFO'], '/')
req = Request.blank('/bucket/obj1', req = Request.blank('/bucket/obj1',
@ -710,7 +710,7 @@ class TestRequest(S3ApiTestCase):
sigv4_req = SigV4Request(req.environ) sigv4_req = SigV4Request(req.environ)
uri = sigv4_req._canonical_uri() uri = sigv4_req._canonical_uri()
self.assertEqual(uri, '/bucket/obj1') self.assertEqual(uri, b'/bucket/obj1')
self.assertEqual(req.environ['PATH_INFO'], '/bucket/obj1') self.assertEqual(req.environ['PATH_INFO'], '/bucket/obj1')
@patch.object(S3Request, '_validate_dates', lambda *a: None) @patch.object(S3Request, '_validate_dates', lambda *a: None)
@ -724,12 +724,12 @@ class TestRequest(S3ApiTestCase):
'bWq2s1WEIj+Ydj0vQ697zp+IXMU='), 'bWq2s1WEIj+Ydj0vQ697zp+IXMU='),
}) })
sigv2_req = S3Request(req.environ, storage_domain='s3.amazonaws.com') sigv2_req = S3Request(req.environ, storage_domain='s3.amazonaws.com')
expected_sts = '\n'.join([ expected_sts = b'\n'.join([
'GET', b'GET',
'', b'',
'', b'',
'Tue, 27 Mar 2007 19:36:42 +0000', b'Tue, 27 Mar 2007 19:36:42 +0000',
'/johnsmith/photos/puppy.jpg', b'/johnsmith/photos/puppy.jpg',
]) ])
self.assertEqual(expected_sts, sigv2_req._string_to_sign()) self.assertEqual(expected_sts, sigv2_req._string_to_sign())
self.assertTrue(sigv2_req.check_signature(secret)) self.assertTrue(sigv2_req.check_signature(secret))
@ -743,12 +743,12 @@ class TestRequest(S3ApiTestCase):
'MyyxeRY7whkBe+bq8fHCL/2kKUg='), 'MyyxeRY7whkBe+bq8fHCL/2kKUg='),
}) })
sigv2_req = S3Request(req.environ, storage_domain='s3.amazonaws.com') sigv2_req = S3Request(req.environ, storage_domain='s3.amazonaws.com')
expected_sts = '\n'.join([ expected_sts = b'\n'.join([
'PUT', b'PUT',
'', b'',
'image/jpeg', b'image/jpeg',
'Tue, 27 Mar 2007 21:15:45 +0000', b'Tue, 27 Mar 2007 21:15:45 +0000',
'/johnsmith/photos/puppy.jpg', b'/johnsmith/photos/puppy.jpg',
]) ])
self.assertEqual(expected_sts, sigv2_req._string_to_sign()) self.assertEqual(expected_sts, sigv2_req._string_to_sign())
self.assertTrue(sigv2_req.check_signature(secret)) self.assertTrue(sigv2_req.check_signature(secret))
@ -763,12 +763,12 @@ class TestRequest(S3ApiTestCase):
'htDYFYduRNen8P9ZfE/s9SuKy0U='), 'htDYFYduRNen8P9ZfE/s9SuKy0U='),
}) })
sigv2_req = S3Request(req.environ, storage_domain='s3.amazonaws.com') sigv2_req = S3Request(req.environ, storage_domain='s3.amazonaws.com')
expected_sts = '\n'.join([ expected_sts = b'\n'.join([
'GET', b'GET',
'', b'',
'', b'',
'Tue, 27 Mar 2007 19:42:41 +0000', b'Tue, 27 Mar 2007 19:42:41 +0000',
'/johnsmith/', b'/johnsmith/',
]) ])
self.assertEqual(expected_sts, sigv2_req._string_to_sign()) self.assertEqual(expected_sts, sigv2_req._string_to_sign())
self.assertTrue(sigv2_req.check_signature(secret)) self.assertTrue(sigv2_req.check_signature(secret))
@ -846,7 +846,7 @@ class TestHashingInput(S3ApiTestCase):
self.assertEqual(b'1234', wrapped.read(4)) self.assertEqual(b'1234', wrapped.read(4))
self.assertEqual(b'56', wrapped.read(2)) self.assertEqual(b'56', wrapped.read(2))
# even though the hash matches, there was more data than we expected # even though the hash matches, there was more data than we expected
with self.assertRaises(swob.Response) as raised: with self.assertRaises(swob.HTTPException) as raised:
wrapped.read(3) wrapped.read(3)
self.assertEqual(raised.exception.status, '422 Unprocessable Entity') self.assertEqual(raised.exception.status, '422 Unprocessable Entity')
# the error causes us to close the input # the error causes us to close the input
@ -859,7 +859,7 @@ class TestHashingInput(S3ApiTestCase):
self.assertEqual(b'1234', wrapped.read(4)) self.assertEqual(b'1234', wrapped.read(4))
self.assertEqual(b'56', wrapped.read(2)) self.assertEqual(b'56', wrapped.read(2))
# even though the hash matches, there was more data than we expected # even though the hash matches, there was more data than we expected
with self.assertRaises(swob.Response) as raised: with self.assertRaises(swob.HTTPException) as raised:
wrapped.read(4) wrapped.read(4)
self.assertEqual(raised.exception.status, '422 Unprocessable Entity') self.assertEqual(raised.exception.status, '422 Unprocessable Entity')
self.assertTrue(wrapped._input.closed) self.assertTrue(wrapped._input.closed)
@ -870,14 +870,14 @@ class TestHashingInput(S3ApiTestCase):
hashlib.md5(raw).hexdigest()) hashlib.md5(raw).hexdigest())
self.assertEqual(b'1234', wrapped.read(4)) self.assertEqual(b'1234', wrapped.read(4))
self.assertEqual(b'5678', wrapped.read(4)) self.assertEqual(b'5678', wrapped.read(4))
with self.assertRaises(swob.Response) as raised: with self.assertRaises(swob.HTTPException) as raised:
wrapped.read(4) wrapped.read(4)
self.assertEqual(raised.exception.status, '422 Unprocessable Entity') self.assertEqual(raised.exception.status, '422 Unprocessable Entity')
self.assertTrue(wrapped._input.closed) self.assertTrue(wrapped._input.closed)
def test_empty_bad_hash(self): def test_empty_bad_hash(self):
wrapped = HashingInput(BytesIO(b''), 0, hashlib.sha256, 'nope') wrapped = HashingInput(BytesIO(b''), 0, hashlib.sha256, 'nope')
with self.assertRaises(swob.Response) as raised: with self.assertRaises(swob.HTTPException) as raised:
wrapped.read(3) wrapped.read(3)
self.assertEqual(raised.exception.status, '422 Unprocessable Entity') self.assertEqual(raised.exception.status, '422 Unprocessable Entity')
# the error causes us to close the input # the error causes us to close the input

2
test/unit/common/middleware/s3api/test_service.py
View File

@ -135,7 +135,7 @@ class TestS3ApiService(S3ApiTestCase):
self.assertEqual(len(names), len(expected)) self.assertEqual(len(names), len(expected))
for i in expected: for i in expected:
self.assertTrue(i[0] in names) self.assertIn(i[0], names)
def _test_service_GET_for_check_bucket_owner(self, buckets): def _test_service_GET_for_check_bucket_owner(self, buckets):
self.s3api.conf.check_bucket_owner = True self.s3api.conf.check_bucket_owner = True

2
test/unit/common/test_internal_client.py
View File

@ -1289,7 +1289,7 @@ class TestInternalClient(unittest.TestCase):
def fake_app(self, env, start_response): def fake_app(self, env, start_response):
start_response('404 Not Found', []) start_response('404 Not Found', [])
return ['one\ntwo\nthree'] return [b'one\ntwo\nthree']
client = InternalClient() client = InternalClient()
lines = [] lines = []

14
test/unit/common/test_swob.py
View File

@ -670,7 +670,7 @@ class TestRequest(unittest.TestCase):
def test_app(environ, start_response): def test_app(environ, start_response):
start_response('401 Unauthorized', []) start_response('401 Unauthorized', [])
return ['hi'] return [b'hi']
# Request environment contains valid account in path # Request environment contains valid account in path
req = swift.common.swob.Request.blank('/v1/account-name') req = swift.common.swob.Request.blank('/v1/account-name')
@ -692,7 +692,7 @@ class TestRequest(unittest.TestCase):
def test_app(environ, start_response): def test_app(environ, start_response):
start_response('401 Unauthorized', []) start_response('401 Unauthorized', [])
return ['hi'] return [b'hi']
# Request environment contains bad path # Request environment contains bad path
req = swift.common.swob.Request.blank('/random') req = swift.common.swob.Request.blank('/random')
@ -706,7 +706,7 @@ class TestRequest(unittest.TestCase):
def test_app(environ, start_response): def test_app(environ, start_response):
start_response('401 Unauthorized', []) start_response('401 Unauthorized', [])
return ['no creds in request'] return [b'no creds in request']
# Request to get token # Request to get token
req = swift.common.swob.Request.blank('/v1.0/auth') req = swift.common.swob.Request.blank('/v1.0/auth')
@ -729,7 +729,7 @@ class TestRequest(unittest.TestCase):
def test_app(environ, start_response): def test_app(environ, start_response):
start_response('401 Unauthorized', { start_response('401 Unauthorized', {
'Www-Authenticate': 'Me realm="whatever"'}) 'Www-Authenticate': 'Me realm="whatever"'})
return ['no creds in request'] return [b'no creds in request']
# Auth middleware sets own Www-Authenticate # Auth middleware sets own Www-Authenticate
req = swift.common.swob.Request.blank('/auth/v1.0') req = swift.common.swob.Request.blank('/auth/v1.0')
@ -743,7 +743,7 @@ class TestRequest(unittest.TestCase):
def test_app(environ, start_response): def test_app(environ, start_response):
start_response('401 Unauthorized', []) start_response('401 Unauthorized', [])
return ['hi'] return [b'hi']
hacker = 'account-name\n\n<b>foo<br>' # url injection test hacker = 'account-name\n\n<b>foo<br>' # url injection test
quoted_hacker = quote(hacker) quoted_hacker = quote(hacker)
@ -766,7 +766,7 @@ class TestRequest(unittest.TestCase):
# Other status codes should not have WWW-Authenticate in response # Other status codes should not have WWW-Authenticate in response
def test_app(environ, start_response): def test_app(environ, start_response):
start_response('200 OK', []) start_response('200 OK', [])
return ['hi'] return [b'hi']
req = swift.common.swob.Request.blank('/') req = swift.common.swob.Request.blank('/')
resp = req.get_response(test_app) resp = req.get_response(test_app)
@ -1763,7 +1763,7 @@ class TestConditionalIfMatch(unittest.TestCase):
def fake_app_404(environ, start_response): def fake_app_404(environ, start_response):
start_response('404 Not Found', []) start_response('404 Not Found', [])
return ['hi'] return [b'hi']
req = swift.common.swob.Request.blank( req = swift.common.swob.Request.blank(
'/', headers={'If-Match': '*'}) '/', headers={'If-Match': '*'})

2
tox.ini
View File

@ -40,7 +40,7 @@ commands =
test/unit/account \ test/unit/account \
test/unit/cli \ test/unit/cli \
test/unit/common/middleware/crypto \ test/unit/common/middleware/crypto \
test/unit/common/middleware/s3api/test_s3token.py \ test/unit/common/middleware/s3api/ \
test/unit/common/middleware/test_account_quotas.py \ test/unit/common/middleware/test_account_quotas.py \
test/unit/common/middleware/test_acl.py \ test/unit/common/middleware/test_acl.py \
test/unit/common/middleware/test_catch_errors.py \ test/unit/common/middleware/test_catch_errors.py \