openstack/swift
Code Issues Proposed changes

s3api: Change default location to us-east-1

Browse Source 
This is more likely to be the default region that a client would try for
v4 signatures.

UpgradeImpact:
==============

Deployers with clusters that relied on the old implicit default
location of US should explicitly set

    location = US

in the [filter:s3api] section of proxy-server.conf before upgrading.

Change-Id: Ib6659a7ad2bd58d711002125e7820f6e86383be8
This commit is contained in:
Tim Burke 2018-11-09 15:46:06 -08:00
parent 168dc91bd9
commit 692a03473f
10 changed files with 93 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
etc/proxy-server.conf-sample
View File

@ -484,7 +484,7 @@ use = egg:swift#s3api
# Set a region name of your Swift cluster. Note that the s3api doesn't choose
# a region of the newly created bucket. This value is used for the
# GET Bucket location API and v4 signatures calculation.
# location = US
# location = us-east-1
#
# Set whether to enforce DNS-compliant bucket names. Note that S3 enforces
# these conventions in all regions except the US Standard region.

2
swift/common/middleware/s3api/controllers/location.py
View File

@ -35,7 +35,7 @@ class LocationController(Controller):
req.get_response(self.app, method='HEAD')
elem = Element('LocationConstraint')
if self.conf.location != 'US':
if self.conf.location != 'us-east-1':
elem.text = self.conf.location
body = tostring(elem)

2
swift/common/middleware/s3api/s3api.py
View File

@ -186,7 +186,7 @@ class S3ApiMiddleware(object):
# Set default values if they are not configured
self.conf.allow_no_owner = config_true_value(
conf.get('allow_no_owner', False))
self.conf.location = conf.get('location', 'US')
self.conf.location = conf.get('location', 'us-east-1')
self.conf.dns_compliant_bucket_names = config_true_value(
conf.get('dns_compliant_bucket_names', True))
self.conf.max_bucket_listing = config_positive_int_value(

8
swift/common/middleware/s3api/s3request.py
View File

@ -461,8 +461,8 @@ class S3Request(swob.Request):
bucket_acl = _header_acl_property('container')
object_acl = _header_acl_property('object')
def __init__(self, env, app=None, slo_enabled=True,
storage_domain='', location='US', force_request_log=False,
def __init__(self, env, app=None, slo_enabled=True, storage_domain='',
location='us-east-1', force_request_log=False,
dns_compliant_bucket_names=True, allow_multipart_uploads=True,
allow_no_owner=False):
# NOTE: app and allow_no_owner are not used by this class, need for
@ -1397,8 +1397,8 @@ class S3AclRequest(S3Request):
"""
S3Acl request object.
"""
def __init__(self, env, app, slo_enabled=True,
storage_domain='', location='US', force_request_log=False,
def __init__(self, env, app, slo_enabled=True, storage_domain='',
location='us-east-1', force_request_log=False,
dns_compliant_bucket_names=True, allow_multipart_uploads=True,
allow_no_owner=False):
super(S3AclRequest, self).__init__(

2
test/functional/s3api/s3_test_client.py
View File

@ -59,7 +59,7 @@ class Connection(object):
S3Connection(aws_access_key, aws_secret_key, is_secure=False,
host=self.host, port=self.port,
calling_format=OrdinaryCallingFormat())
self.conn.auth_region_name = 'US'
self.conn.auth_region_name = 'us-east-1'
def reset(self):
"""

2
test/functional/s3api/test_bucket.py
View File

@ -191,7 +191,7 @@ class TestS3ApiBucket(S3ApiBase):
def test_put_bucket_with_LocationConstraint(self):
bucket = 'bucket'
xml = self._gen_location_xml('US')
xml = self._gen_location_xml(self.conn.conn.auth_region_name)
status, headers, body = \
self.conn.make_request('PUT', bucket, body=xml)
self.assertEqual(status, 200)

2
test/unit/common/middleware/s3api/__init__.py
View File

@ -59,7 +59,7 @@ class S3ApiTestCase(unittest.TestCase):
# setup default config
self.conf = Config({
'allow_no_owner': False,
'location': 'US',
'location': 'us-east-1',
'dns_compliant_bucket_names': True,
'max_bucket_listing': 1000,
'max_parts_listing': 1000,

2
test/unit/common/middleware/s3api/test_bucket.py
View File

@ -597,7 +597,7 @@ class TestS3ApiBucket(S3ApiTestCase):
def _test_bucket_PUT_with_location(self, root_element):
elem = Element(root_element)
SubElement(elem, 'LocationConstraint').text = 'US'
SubElement(elem, 'LocationConstraint').text = 'us-east-1'
xml = tostring(elem)
req = Request.blank('/bucket',

143
test/unit/common/middleware/s3api/test_s3api.py
View File

@ -319,7 +319,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
req = Request.blank(
'/bucket/object'
'?X-Amz-Algorithm=AWS4-HMAC-SHA256'
'&X-Amz-Credential=test:tester/%s/US/s3/aws4_request'
'&X-Amz-Credential=test:tester/%s/us-east-1/s3/aws4_request'
'&X-Amz-Date=%s'
'&X-Amz-Expires=1000'
'&X-Amz-SignedHeaders=host'
@ -358,54 +358,58 @@ class TestS3ApiMiddleware(S3ApiTestCase):
self.assertIn(extra, body)
dt = self.get_v4_amz_date_header().split('T', 1)[0]
test('test:tester/not-a-date/US/s3/aws4_request',
test('test:tester/not-a-date/us-east-1/s3/aws4_request',
'Invalid credential date "not-a-date". This date is not the same '
'as X-Amz-Date: "%s".' % dt)
test('test:tester/%s/not-US/s3/aws4_request' % dt,
test('test:tester/%s/us-west-1/s3/aws4_request' % dt,
"Error parsing the X-Amz-Credential parameter; the region "
"'not-US' is wrong; expecting 'US'", '<Region>US</Region>')
test('test:tester/%s/US/not-s3/aws4_request' % dt,
"'us-west-1' is wrong; expecting 'us-east-1'",
'<Region>us-east-1</Region>')
test('test:tester/%s/us-east-1/not-s3/aws4_request' % dt,
'Error parsing the X-Amz-Credential parameter; incorrect service '
'"not-s3". This endpoint belongs to "s3".')
test('test:tester/%s/US/s3/not-aws4_request' % dt,
test('test:tester/%s/us-east-1/s3/not-aws4_request' % dt,
'Error parsing the X-Amz-Credential parameter; incorrect '
'terminal "not-aws4_request". This endpoint uses "aws4_request".')
def test_signed_urls_v4_missing_x_amz_date(self):
req = Request.blank('/bucket/object'
'?X-Amz-Algorithm=AWS4-HMAC-SHA256'
'&X-Amz-Credential=test/20T20Z/US/s3/aws4_request'
'&X-Amz-Expires=1000'
'&X-Amz-SignedHeaders=host'
'&X-Amz-Signature=X',
environ={'REQUEST_METHOD': 'GET'})
req = Request.blank(
'/bucket/object'
'?X-Amz-Algorithm=AWS4-HMAC-SHA256'
'&X-Amz-Credential=test/20T20Z/us-east-1/s3/aws4_request'
'&X-Amz-Expires=1000'
'&X-Amz-SignedHeaders=host'
'&X-Amz-Signature=X',
environ={'REQUEST_METHOD': 'GET'})
req.content_type = 'text/plain'
status, headers, body = self.call_s3api(req)
self.assertEqual(self._get_error_code(body), 'AccessDenied')
def test_signed_urls_v4_invalid_algorithm(self):
req = Request.blank('/bucket/object'
'?X-Amz-Algorithm=FAKE'
'&X-Amz-Credential=test/20T20Z/US/s3/aws4_request'
'&X-Amz-Date=%s'
'&X-Amz-Expires=1000'
'&X-Amz-SignedHeaders=host'
'&X-Amz-Signature=X' %
self.get_v4_amz_date_header(),
environ={'REQUEST_METHOD': 'GET'})
req = Request.blank(
'/bucket/object'
'?X-Amz-Algorithm=FAKE'
'&X-Amz-Credential=test/20T20Z/us-east-1/s3/aws4_request'
'&X-Amz-Date=%s'
'&X-Amz-Expires=1000'
'&X-Amz-SignedHeaders=host'
'&X-Amz-Signature=X' %
self.get_v4_amz_date_header(),
environ={'REQUEST_METHOD': 'GET'})
req.content_type = 'text/plain'
status, headers, body = self.call_s3api(req)
self.assertEqual(self._get_error_code(body), 'InvalidArgument')
def test_signed_urls_v4_missing_signed_headers(self):
req = Request.blank('/bucket/object'
'?X-Amz-Algorithm=AWS4-HMAC-SHA256'
'&X-Amz-Credential=test/20T20Z/US/s3/aws4_request'
'&X-Amz-Date=%s'
'&X-Amz-Expires=1000'
'&X-Amz-Signature=X' %
self.get_v4_amz_date_header(),
environ={'REQUEST_METHOD': 'GET'})
req = Request.blank(
'/bucket/object'
'?X-Amz-Algorithm=AWS4-HMAC-SHA256'
'&X-Amz-Credential=test/20T20Z/us-east-1/s3/aws4_request'
'&X-Amz-Date=%s'
'&X-Amz-Expires=1000'
'&X-Amz-Signature=X' %
self.get_v4_amz_date_header(),
environ={'REQUEST_METHOD': 'GET'})
req.content_type = 'text/plain'
status, headers, body = self.call_s3api(req)
self.assertEqual(self._get_error_code(body),
@ -426,14 +430,15 @@ class TestS3ApiMiddleware(S3ApiTestCase):
self.assertEqual(self._get_error_code(body), 'AccessDenied')
def test_signed_urls_v4_missing_signature(self):
req = Request.blank('/bucket/object'
'?X-Amz-Algorithm=AWS4-HMAC-SHA256'
'&X-Amz-Credential=test/20T20Z/US/s3/aws4_request'
'&X-Amz-Date=%s'
'&X-Amz-Expires=1000'
'&X-Amz-SignedHeaders=host' %
self.get_v4_amz_date_header(),
environ={'REQUEST_METHOD': 'GET'})
req = Request.blank(
'/bucket/object'
'?X-Amz-Algorithm=AWS4-HMAC-SHA256'
'&X-Amz-Credential=test/20T20Z/us-east-1/s3/aws4_request'
'&X-Amz-Date=%s'
'&X-Amz-Expires=1000'
'&X-Amz-SignedHeaders=host' %
self.get_v4_amz_date_header(),
environ={'REQUEST_METHOD': 'GET'})
req.content_type = 'text/plain'
status, headers, body = self.call_s3api(req)
self.assertEqual(self._get_error_code(body), 'AccessDenied')
@ -710,7 +715,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
headers = {
'Authorization':
'AWS4-HMAC-SHA256 '
'Credential=test:tester/%s/US/s3/aws4_request, '
'Credential=test:tester/%s/us-east-1/s3/aws4_request, '
'SignedHeaders=host;x-amz-date,'
'Signature=X' % self.get_v4_amz_date_header().split('T', 1)[0],
'X-Amz-Date': self.get_v4_amz_date_header(),
@ -729,7 +734,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
headers = {
'Authorization':
'AWS4-HMAC-SHA256 '
'Credential=test:tester/20130524/US/s3/aws4_request, '
'Credential=test:tester/20130524/us-east-1/s3/aws4_request, '
'SignedHeaders=host;range;x-amz-date,'
'Signature=X',
'X-Amz-Content-SHA256': '0123456789'}
@ -745,7 +750,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
headers = {
'Authorization':
'AWS4-HMAC-SHA256 '
'Credential=test:tester/%s/US/s3/aws4_request, '
'Credential=test:tester/%s/us-east-1/s3/aws4_request, '
'SignedHeaders=host;x-amz-date,'
'Signature=X' % self.get_v4_amz_date_header().split('T', 1)[0],
'X-Amz-Date': self.get_v4_amz_date_header()}
@ -779,25 +784,26 @@ class TestS3ApiMiddleware(S3ApiTestCase):
'Signature=X')
test(auth_str, 'AccessDenied', 'Access Denied.')
auth_str = ('AWS4-HMAC-SHA256 '
'Credential=test:tester/20130524/US/s3/aws4_request, '
'Signature=X')
auth_str = (
'AWS4-HMAC-SHA256 '
'Credential=test:tester/20130524/us-east-1/s3/aws4_request, '
'Signature=X')
test(auth_str, 'AuthorizationHeaderMalformed',
'The authorization header is malformed; the authorization '
'header requires three components: Credential, SignedHeaders, '
'and Signature.')
auth_str = ('AWS4-HMAC-SHA256 '
'Credential=test:tester/%s/not-US/s3/aws4_request, '
'Credential=test:tester/%s/us-west-2/s3/aws4_request, '
'Signature=X, SignedHeaders=host;x-amz-date' %
self.get_v4_amz_date_header().split('T', 1)[0])
test(auth_str, 'AuthorizationHeaderMalformed',
"The authorization header is malformed; "
"the region 'not-US' is wrong; expecting 'US'",
'<Region>US</Region>')
"the region 'us-west-2' is wrong; expecting 'us-east-1'",
'<Region>us-east-1</Region>')
auth_str = ('AWS4-HMAC-SHA256 '
'Credential=test:tester/%s/US/not-s3/aws4_request, '
'Credential=test:tester/%s/us-east-1/not-s3/aws4_request, '
'Signature=X, SignedHeaders=host;x-amz-date' %
self.get_v4_amz_date_header().split('T', 1)[0])
test(auth_str, 'AuthorizationHeaderMalformed',
@ -805,7 +811,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
'incorrect service "not-s3". This endpoint belongs to "s3".')
auth_str = ('AWS4-HMAC-SHA256 '
'Credential=test:tester/%s/US/s3/not-aws4_request, '
'Credential=test:tester/%s/us-east-1/s3/not-aws4_request, '
'Signature=X, SignedHeaders=host;x-amz-date' %
self.get_v4_amz_date_header().split('T', 1)[0])
test(auth_str, 'AuthorizationHeaderMalformed',
@ -813,9 +819,10 @@ class TestS3ApiMiddleware(S3ApiTestCase):
'incorrect terminal "not-aws4_request". '
'This endpoint uses "aws4_request".')
auth_str = ('AWS4-HMAC-SHA256 '
'Credential=test:tester/20130524/US/s3/aws4_request, '
'SignedHeaders=host;x-amz-date')
auth_str = (
'AWS4-HMAC-SHA256 '
'Credential=test:tester/20130524/us-east-1/s3/aws4_request, '
'SignedHeaders=host;x-amz-date')
test(auth_str, 'AccessDenied', 'Access Denied.')
def test_canonical_string_v4(self):
@ -835,7 +842,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
'27ae41e4649b934ca495991b7852b855',
'HTTP_AUTHORIZATION':
'AWS4-HMAC-SHA256 '
'Credential=X:Y/20110909/US/s3/aws4_request, '
'Credential=X:Y/20110909/us-east-1/s3/aws4_request, '
'SignedHeaders=content-md5;content-type;date, '
'Signature=x',
}
@ -1007,7 +1014,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
headers = {
'Authorization':
'AWS4-HMAC-SHA256 '
'Credential=test/20130524/US/s3/aws4_request_A, '
'Credential=test/20130524/us-east-1/s3/aws4_request_A, '
'SignedHeaders=hostA;rangeA;x-amz-dateA,'
'Signature=X',
'X-Amz-Date': self.get_v4_amz_date_header(),
@ -1015,13 +1022,14 @@ class TestS3ApiMiddleware(S3ApiTestCase):
# and then, different auth info (Credential, SignedHeaders, Signature)
# in query
req = Request.blank('/bucket/object'
'?X-Amz-Algorithm=AWS4-HMAC-SHA256'
'&X-Amz-Credential=test/20T20Z/US/s3/aws4_requestB'
'&X-Amz-SignedHeaders=hostB'
'&X-Amz-Signature=Y',
environ={'REQUEST_METHOD': 'GET'},
headers=headers)
req = Request.blank(
'/bucket/object'
'?X-Amz-Algorithm=AWS4-HMAC-SHA256'
'&X-Amz-Credential=test/20T20Z/us-east-1/s3/aws4_requestB'
'&X-Amz-SignedHeaders=hostB'
'&X-Amz-Signature=Y',
environ={'REQUEST_METHOD': 'GET'},
headers=headers)
req.content_type = 'text/plain'
status, headers, body = self.call_s3api(req)
# FIXME: should this failed as 400 or pass via query auth?
@ -1029,12 +1037,13 @@ class TestS3ApiMiddleware(S3ApiTestCase):
self.assertEqual(status.split()[0], '403', body)
# But if we are missing Signature in query param
req = Request.blank('/bucket/object'
'?X-Amz-Algorithm=AWS4-HMAC-SHA256'
'&X-Amz-Credential=test/20T20Z/US/s3/aws4_requestB'
'&X-Amz-SignedHeaders=hostB',
environ={'REQUEST_METHOD': 'GET'},
headers=headers)
req = Request.blank(
'/bucket/object'
'?X-Amz-Algorithm=AWS4-HMAC-SHA256'
'&X-Amz-Credential=test/20T20Z/us-east-1/s3/aws4_requestB'
'&X-Amz-SignedHeaders=hostB',
environ={'REQUEST_METHOD': 'GET'},
headers=headers)
req.content_type = 'text/plain'
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '403', body)

12
test/unit/common/middleware/s3api/test_s3request.py
View File

@ -400,7 +400,7 @@ class TestRequest(S3ApiTestCase):
headers = {
'Authorization':
'AWS4-HMAC-SHA256 '
'Credential=test/%s/US/s3/aws4_request, '
'Credential=test/%s/us-east-1/s3/aws4_request, '
'SignedHeaders=%s,'
'Signature=X' % (
self.get_v4_amz_date_header().split('T', 1)[0],
@ -558,7 +558,7 @@ class TestRequest(S3ApiTestCase):
headers = {
'Authorization':
'AWS4-HMAC-SHA256 '
'Credential=test/%s/US/s3/aws4_request, '
'Credential=test/%s/us-east-1/s3/aws4_request, '
'SignedHeaders=host;x-amz-content-sha256;x-amz-date,'
'Signature=X' % self.get_v4_amz_date_header().split('T', 1)[0],
'X-Amz-Content-SHA256': '0123456789',
@ -578,7 +578,7 @@ class TestRequest(S3ApiTestCase):
headers = {
'Authorization':
'AWS4-HMAC-SHA256 '
'Credential=test/%s/US/s3/aws4_request, '
'Credential=test/%s/us-east-1/s3/aws4_request, '
'SignedHeaders=host;x-amz-content-sha256,'
'Signature=X' % self.get_v4_amz_date_header().split('T', 1)[0],
'X-Amz-Content-SHA256': '0123456789',
@ -597,7 +597,7 @@ class TestRequest(S3ApiTestCase):
headers = {
'Authorization':
'AWS4-HMAC-SHA256 '
'Credential=test/%s/US/s3/aws4_request, '
'Credential=test/%s/us-east-1/s3/aws4_request, '
'SignedHeaders=host;x-amz-content-sha256;x-amz-date,'
'Signature=X' % self.get_v4_amz_date_header().split('T', 1)[0],
'X-Amz-Content-SHA256': '0123456789',
@ -661,7 +661,7 @@ class TestRequest(S3ApiTestCase):
headers = {
'Authorization':
'AWS4-HMAC-SHA256 '
'Credential=test/%s/US/s3/aws4_request, '
'Credential=test/%s/us-east-1/s3/aws4_request, '
'SignedHeaders=host;x-amz-content-sha256;x-amz-date,'
'Signature=X' % self.get_v4_amz_date_header().split('T', 1)[0],
'X-Amz-Content-SHA256': '0123456789',
@ -794,7 +794,7 @@ class TestRequest(S3ApiTestCase):
req = Request.blank('/photos/puppy.jpg', headers={
'Authorization':
'AWS4-HMAC-SHA256 '
'Credential=test/%s/US/s3/aws4_request, '
'Credential=test/%s/us-east-1/s3/aws4_request, '
'SignedHeaders=host;x-amz-content-sha256;x-amz-date,'
'Signature=X' % amz_date_header.split('T', 1)[0],
'X-Amz-Content-SHA256': '0123456789',