x/swift3
Code Issues Proposed changes

Merge "Add support for more characters in header keys"

Browse Source
This commit is contained in:
Jenkins 2017-10-13 07:28:49 +00:00 committed by Gerrit Code Review
commit 9997bc1953
4 changed files with 108 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

99
swift3/request.py
View File

@ -14,6 +14,7 @@
# limitations under the License.
import base64
from collections import defaultdict
from email.header import Header
from hashlib import sha1, sha256, md5
import hmac
@ -266,9 +267,18 @@ class SigV4Mixin(object):
:return : dict of headers to sign, the keys are all lower case
"""
headers_lower_dict = dict(
(k.lower().strip(), ' '.join(_header_strip(v or '').split()))
for (k, v) in six.iteritems(self.headers))
if 'headers_raw' in self.environ: # eventlet >= 0.19.0
# See https://github.com/eventlet/eventlet/commit/67ec999
headers_lower_dict = defaultdict(list)
for key, value in self.environ['headers_raw']:
headers_lower_dict[key.lower().strip()].append(
' '.join(_header_strip(value or '').split()))
headers_lower_dict = {k: ','.join(v)
for k, v in headers_lower_dict.items()}
else: # mostly-functional fallback
headers_lower_dict = dict(
(k.lower().strip(), ' '.join(_header_strip(v or '').split()))
for (k, v) in six.iteritems(self.headers))
if 'host' in headers_lower_dict and re.match(
'Boto/2.[0-9].[0-2]',
@ -280,7 +290,7 @@ class SigV4Mixin(object):
headers_lower_dict['host'].split(':')[0]
headers_to_sign = [
(key, value) for key, value in headers_lower_dict.items()
(key, value) for key, value in sorted(headers_lower_dict.items())
if key in self._signed_headers]
if len(headers_to_sign) != len(self._signed_headers):
@ -291,7 +301,7 @@ class SigV4Mixin(object):
# process.
raise SignatureDoesNotMatch()
return dict(headers_to_sign)
return headers_to_sign
def _canonical_uri(self):
"""
@ -329,13 +339,12 @@ class SigV4Mixin(object):
# host:iam.amazonaws.com
# x-amz-date:20150830T123600Z
headers_to_sign = self._headers_to_sign()
cr.append('\n'.join(
['%s:%s' % (key, value) for key, value in
sorted(headers_to_sign.items())]) + '\n')
cr.append(''.join('%s:%s\n' % (key, value)
for key, value in headers_to_sign))
# 5. Add signed headers into canonical request like
# content-type;host;x-amz-date
cr.append(';'.join(sorted(headers_to_sign)))
cr.append(';'.join(k for k, v in headers_to_sign))
# 6. Add payload string at the tail
if 'X-Amz-Credential' in self.params:
@ -800,9 +809,20 @@ class Request(swob.Request):
_header_strip(self.headers.get('Content-MD5')) or '',
_header_strip(self.headers.get('Content-Type')) or '']
for amz_header in sorted((key.lower() for key in self.headers
if key.lower().startswith('x-amz-'))):
amz_headers[amz_header] = self.headers[amz_header]
if 'headers_raw' in self.environ: # eventlet >= 0.19.0
# See https://github.com/eventlet/eventlet/commit/67ec999
amz_headers = defaultdict(list)
for key, value in self.environ['headers_raw']:
key = key.lower()
if not key.startswith('x-amz-'):
continue
amz_headers[key.strip()].append(value.strip())
amz_headers = dict((key, ','.join(value))
for key, value in amz_headers.items())
else: # mostly-functional fallback
amz_headers = dict((key.lower(), value)
for key, value in self.headers.items()
if key.lower().startswith('x-amz-'))
if self._is_header_auth:
if 'x-amz-date' in amz_headers:
@ -816,8 +836,8 @@ class Request(swob.Request):
# but as a sanity check...
raise AccessDenied()
for k in sorted(key.lower() for key in amz_headers):
buf.append("%s:%s" % (k, amz_headers[k]))
for key, value in sorted(amz_headers.items()):
buf.append("%s:%s" % (key, value))
path = self._canonical_uri()
if self.query_string:
@ -903,15 +923,48 @@ class Request(swob.Request):
env = self.environ.copy()
for key in self.environ:
if key.startswith('HTTP_X_AMZ_META_'):
if not(set(env[key]).issubset(string.printable)):
env[key] = Header(env[key], 'UTF-8').encode()
if env[key].startswith('=?utf-8?q?'):
env[key] = '=?UTF-8?Q?' + env[key][10:]
elif env[key].startswith('=?utf-8?b?'):
env[key] = '=?UTF-8?B?' + env[key][10:]
env['HTTP_X_OBJECT_META_' + key[16:]] = env[key]
def sanitize(value):
if set(value).issubset(string.printable):
return value
value = Header(value, 'UTF-8').encode()
if value.startswith('=?utf-8?q?'):
return '=?UTF-8?Q?' + value[10:]
elif value.startswith('=?utf-8?b?'):
return '=?UTF-8?B?' + value[10:]
else:
return value
if 'headers_raw' in env: # eventlet >= 0.19.0
# See https://github.com/eventlet/eventlet/commit/67ec999
for key, value in env['headers_raw']:
if not key.lower().startswith('x-amz-meta-'):
continue
# AWS ignores user-defined headers with these characters
if any(c in key for c in ' "),/;<=>?@[\\]{}'):
# NB: apparently, '(' *is* allowed
continue
# Note that this may have already been deleted, e.g. if the
# client sent multiple headers with the same name, or both
# x-amz-meta-foo-bar and x-amz-meta-foo_bar
env.pop('HTTP_' + key.replace('-', '_').upper(), None)
# Need to preserve underscores. Since we know '=' can't be
# present, quoted-printable seems appropriate.
key = key.replace('_', '=5F').replace('-', '_').upper()
key = 'HTTP_X_OBJECT_META_' + key[11:]
if key in env:
env[key] += ',' + sanitize(value)
else:
env[key] = sanitize(value)
else: # mostly-functional fallback
for key in self.environ:
if not key.startswith('HTTP_X_AMZ_META_'):
continue
# AWS ignores user-defined headers with these characters
if any(c in key for c in ' "),/;<=>?@[\\]{}'):
# NB: apparently, '(' *is* allowed
continue
env['HTTP_X_OBJECT_META_' + key[16:]] = sanitize(env[key])
del env[key]
if 'HTTP_X_AMZ_COPY_SOURCE' in env:

5
swift3/response.py
View File

@ -100,7 +100,10 @@ class Response(ResponseBase, swob.Response):
_key = key.lower()
if _key.startswith('x-object-meta-'):
headers['x-amz-meta-' + _key[14:]] = val
# Note that AWS ignores user-defined headers with '=' in the
# header name. We translated underscores to '=5F' on the way
# in, though.
headers['x-amz-meta-' + _key[14:].replace('=5f', '_')] = val
elif _key in ('content-length', 'content-type',
'content-range', 'content-encoding',
'content-disposition', 'content-language',

21
swift3/test/functional/test_object.py
View File

@ -318,7 +318,9 @@ class TestSwift3Object(Swift3FunctionalTestCase):
self.assertCommonResponseHeaders(headers)
self._assertObjectEtag(self.bucket, obj, etag)
def _test_put_object_headers(self, req_headers):
def _test_put_object_headers(self, req_headers, expected_headers=None):
if expected_headers is None:
expected_headers = req_headers
obj = 'object'
content = 'abcdefghij'
etag = md5(content).hexdigest()
@ -328,7 +330,7 @@ class TestSwift3Object(Swift3FunctionalTestCase):
self.assertEqual(status, 200)
status, headers, body = \
self.conn.make_request('HEAD', self.bucket, obj)
for header, value in req_headers.items():
for header, value in expected_headers.items():
self.assertIn(header.lower(), headers)
self.assertEqual(headers[header.lower()], value)
self.assertCommonResponseHeaders(headers)
@ -339,6 +341,21 @@ class TestSwift3Object(Swift3FunctionalTestCase):
'X-Amz-Meta-Bar': 'foo',
'X-Amz-Meta-Bar2': 'foo2'})
def test_put_object_weird_metadata(self):
req_headers = dict(
('x-amz-meta-' + c, c)
for c in '!"#$%&\'()*+-./<=>?@[\\]^`{|}~')
exp_headers = dict(
('x-amz-meta-' + c, c)
for c in '!#$%&\'(*+-.^`|~')
self._test_put_object_headers(req_headers, exp_headers)
def test_put_object_underscore_in_metadata(self):
# Break this out separately for ease of testing pre-0.19.0 eventlet
self._test_put_object_headers({
'X-Amz-Meta-Foo-Bar': 'baz',
'X-Amz-Meta-Foo_Bar': 'also baz'})
def test_put_object_content_headers(self):
self._test_put_object_headers({
'Content-Type': 'foo/bar',

20
swift3/test/unit/test_request.py
View File

@ -391,8 +391,8 @@ class TestRequest(Swift3TestCase):
'Authorization':
'AWS4-HMAC-SHA256 '
'Credential=test/20130524/US/s3/aws4_request, '
'SignedHeaders=host;%s,'
'Signature=X' % included_header,
'SignedHeaders=%s,'
'Signature=X' % ';'.join(sorted(['host', included_header])),
'X-Amz-Content-SHA256': '0123456789'}
headers.update(date_header)
@ -551,11 +551,10 @@ class TestRequest(Swift3TestCase):
sigv4_req = SigV4Request(req.environ)
headers_to_sign = sigv4_req._headers_to_sign()
self.assertEqual(['host', 'x-amz-content-sha256', 'x-amz-date'],
sorted(headers_to_sign.keys()))
self.assertEqual(headers_to_sign['host'], 'localhost:80')
self.assertEqual(headers_to_sign['x-amz-date'], x_amz_date)
self.assertEqual(headers_to_sign['x-amz-content-sha256'], '0123456789')
self.assertEqual(headers_to_sign, [
('host', 'localhost:80'),
('x-amz-content-sha256', '0123456789'),
('x-amz-date', x_amz_date)])
# no x-amz-date
headers = {
@ -571,10 +570,9 @@ class TestRequest(Swift3TestCase):
sigv4_req = SigV4Request(req.environ)
headers_to_sign = sigv4_req._headers_to_sign()
self.assertEqual(['host', 'x-amz-content-sha256'],
sorted(headers_to_sign.keys()))
self.assertEqual(headers_to_sign['host'], 'localhost:80')
self.assertEqual(headers_to_sign['x-amz-content-sha256'], '0123456789')
self.assertEqual(headers_to_sign, [
('host', 'localhost:80'),
('x-amz-content-sha256', '0123456789')])
# SignedHeaders says, host and x-amz-date included but there is not
# X-Amz-Date header