openstack/swift
Code Issues Proposed changes

s3api: Pass through CORS headers

Browse Source 
This adds support for presigned GET URLs, at least.

Note that there is no support yet for preflight requests, so a whole
bunch of other CORS stuff *doesn't* work (yet). This was just an easy
first step.

Change-Id: I43150a630a2a7620099e6bfecaed3bbe958ba423
This commit is contained in:
Tim Burke 2020-02-27 10:39:38 -08:00
parent c5152ed4d3
commit 81db980690
11 changed files with 371 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.zuul.yaml
View File

@ -272,6 +272,8 @@
description: |
Setup a SAIO dev environment and run ceph-s3tests
timeout: 5400
vars:
s3_acl: yes
pre-run:
- tools/playbooks/common/install_dependencies.yaml
- tools/playbooks/saio_single_node_setup/setup_saio.yaml
@ -315,7 +317,10 @@
description: |
Setup a SAIO dev environment and run Swift's CORS functional tests
timeout: 1200
vars:
s3_acl: no
pre-run:
- tools/playbooks/saio_single_node_setup/add_s3api.yaml
- tools/playbooks/cors/install_selenium.yaml
run: tools/playbooks/cors/run.yaml
post-run: tools/playbooks/cors/post.yaml

77
swift/common/middleware/s3api/s3response.py
View File

@ -46,6 +46,55 @@ class HeaderKeyDict(header_key_dict.HeaderKeyDict):
return s
def translate_swift_to_s3(key, val):
_key = swob.bytes_to_wsgi(swob.wsgi_to_bytes(key).lower())
def translate_meta_key(_key):
if not _key.startswith('x-object-meta-'):
return _key
# Note that AWS allows user-defined metadata with underscores in the
# header, while WSGI (and other protocols derived from CGI) does not
# differentiate between an underscore and a dash. Fortunately,
# eventlet exposes the raw headers from the client, so we could
# translate '_' to '=5F' on the way in. Now, we translate back.
return 'x-amz-meta-' + _key[14:].replace('=5f', '_')
if _key.startswith('x-object-meta-'):
return translate_meta_key(_key), val
elif _key in ('content-length', 'content-type',
'content-range', 'content-encoding',
'content-disposition', 'content-language',
'etag', 'last-modified', 'x-robots-tag',
'cache-control', 'expires'):
return key, val
elif _key == 'x-object-version-id':
return 'x-amz-version-id', val
elif _key == 'x-copied-from-version-id':
return 'x-amz-copy-source-version-id', val
elif _key == 'x-backend-content-type' and \
val == DELETE_MARKER_CONTENT_TYPE:
return 'x-amz-delete-marker', 'true'
elif _key == 'access-control-expose-headers':
exposed_headers = val.split(', ')
exposed_headers.extend([
'x-amz-request-id',
'x-amz-id-2',
])
return 'access-control-expose-headers', ', '.join(
translate_meta_key(h) for h in exposed_headers)
elif _key == 'access-control-allow-methods':
methods = val.split(', ')
try:
methods.remove('COPY') # that's not a thing in S3
except ValueError:
pass # not there? don't worry about it
return key, ', '.join(methods)
elif _key.startswith('access-control-'):
return key, val
# else, drop the header
return None
class S3ResponseBase(object):
"""
Base class for swift3 responses.
@ -98,29 +147,13 @@ class S3Response(S3ResponseBase, swob.Response):
# Handle swift headers
for key, val in sw_headers.items():
_key = swob.bytes_to_wsgi(swob.wsgi_to_bytes(key).lower())
s3_pair = translate_swift_to_s3(key, val)
if s3_pair is None:
continue
headers[s3_pair[0]] = s3_pair[1]
if _key.startswith('x-object-meta-'):
# Note that AWS ignores user-defined headers with '=' in the
# header name. We translated underscores to '=5F' on the way
# in, though.
headers['x-amz-meta-' + _key[14:].replace('=5f', '_')] = val
elif _key in ('content-length', 'content-type',
'content-range', 'content-encoding',
'content-disposition', 'content-language',
'etag', 'last-modified', 'x-robots-tag',
'cache-control', 'expires'):
headers[key] = val
elif _key == 'x-object-version-id':
headers['x-amz-version-id'] = val
elif _key == 'x-copied-from-version-id':
headers['x-amz-copy-source-version-id'] = val
elif _key == 'x-static-large-object':
# for delete slo
self.is_slo = config_true_value(val)
elif _key == 'x-backend-content-type' and \
val == DELETE_MARKER_CONTENT_TYPE:
headers['x-amz-delete-marker'] = 'true'
self.is_slo = config_true_value(sw_headers.get(
'x-static-large-object'))
# Check whether we stored the AWS-style etag on upload
override_etag = s3_sysmeta_headers.get(

27
test/cors/README.rst
View File

@ -41,6 +41,16 @@ environment variables to determine how to connect to Swift:
* ``OS_PASSWORD`` (or ``ST_KEY``)
* ``OS_STORAGE_URL`` (optional)
There are additional environment variables to exercise the S3 API:
* ``S3_ENDPOINT``
* ``S3_USER``
* ``S3_KEY``
.. note::
It is necessary to set `s3_acl = False` in the `[filter:s3api]` section of
your `proxy-server.conf` for all the s3 object tests to pass.
..
TODO: verify that this works with Keystone
@ -54,7 +64,7 @@ To inspect the test results in your local browser, run::
This will create some test containers and object in Swift, start a simple
static site, and emit a URL to visit to run the tests, like::
Serving test at http://localhost:8000/#OS_AUTH_URL=http://saio/auth/v1.0&OS_USERNAME=test:tester&OS_PASSWORD=testing&OS_STORAGE_URL=http://saio/v1/AUTH_test
Serving test at http://localhost:8000/#OS_AUTH_URL=http://saio/auth/v1.0&OS_USERNAME=test:tester&OS_PASSWORD=testing&OS_STORAGE_URL=http://saio/v1/AUTH_test&S3_ENDPOINT=http://saio&S3_USER=test%3Atester&S3_KEY=testing
.. note::
You can use ``--hostname`` and ``--port`` to adjust the origin used.
@ -95,3 +105,18 @@ for the Python bindings for more information about setting this up.
When using selenium, the test runner will try to run tests in Firefox, Chrome,
Safari, Edge, and IE if available; if a browser seems to not be available, its
tests will be skipped.
Updating aws-sdk-js
-------------------
There are tests that exercise CORS over the S3 API; these use a vendored
version of `aws-sdk-js <https://github.com/aws/aws-sdk-js/>`__ that only
covers the S3 service. The current version used is 2.829.0, built on
2021-01-21 by
* visiting https://sdk.amazonaws.com/builder/js/,
* clearing all services,
* explicitly adding AWS.S3,
* clicking "Build" to download,
* saving in the ``test/cors/vendor`` directory, and finally
* updating the version number in ``test/cors/test-s3*.js``.

13
test/cors/harness.js
View File

@ -14,10 +14,17 @@ function makeUrl (path) {
export function MakeRequest (method, path, headers, body, params) {
var url = makeUrl(path)
headers = headers || {}
params = params || {}
// give each request a unique query string to avoid ever fetching from cache
params['cors-test-time'] = Date.now().toString()
params['cors-test-random'] = Math.random().toString()
if (!(
url.searchParams.has('Signature') ||
url.searchParams.has('X-Amz-Signature') ||
'Authorization' in headers
)) {
// give each Swift request a unique query string to avoid ever fetching from cache
params['cors-test-time'] = Date.now().toString()
params['cors-test-random'] = Math.random().toString()
}
for (var key in params) {
url.searchParams.append(key, params[key])
}

2
test/cors/index.html
View File

@ -22,6 +22,7 @@ td:nth-child(2) {
.map(v => v.split('='))
.reduce( (acc, [key, val]) => ({ ...acc, [unescape(key)]: unescape(val) }), {})
console.log(PARAMS)
var _xamzrequire // Needed to be able to import the sdk later
</script>
<script type="module" src="test-info.js"></script>
<script type="module" src="test-account.js"></script>
@ -29,6 +30,7 @@ td:nth-child(2) {
<script type="module" src="test-object.js"></script>
<script type="module" src="test-large-objects.js"></script>
<script type="module" src="test-symlink.js"></script>
<script type="module" src="test-s3-obj.js"></script>
</head>
<body>
<h2>CORS Tests</h2>

3
test/cors/main.py
View File

@ -39,6 +39,9 @@ DEFAULT_ENV = {
'OS_USERNAME': os.environ.get('ST_USER', 'test:tester'),
'OS_PASSWORD': os.environ.get('ST_KEY', 'testing'),
'OS_STORAGE_URL': None,
'S3_ENDPOINT': 'http://localhost:8080',
'S3_USER': 'test:tester',
'S3_KEY': 'testing',
}
ENV = {key: os.environ.get(key, default)
for key, default in DEFAULT_ENV.items()}

177
test/cors/test-s3-obj.js Normal file
View File

@ -0,0 +1,177 @@
/* global PARAMS */
import {
runTests,
MakeRequest,
HasStatus,
HasHeaders,
DoesNotHaveHeaders,
HasNoBody,
BodyHasLength,
CorsBlocked
} from './harness.js'
import './vendor/aws-sdk-2.829.0.min.js'
const AWS = window.AWS
function CheckTransactionIdHeaders (resp) {
return Promise.resolve(resp)
.then(HasHeaders([
'x-amz-request-id',
'x-amz-id-2',
'X-Openstack-Request-Id',
'X-Trans-Id'
]))
.then((resp) => {
const txnId = resp.getResponseHeader('X-Openstack-Request-Id')
return Promise.resolve(resp)
.then(HasHeaders({
'x-amz-request-id': txnId,
'x-amz-id-2': txnId,
'X-Trans-Id': txnId
}))
})
}
function CheckS3Headers (resp) {
return Promise.resolve(resp)
.then(HasHeaders([
'Last-Modified',
'Content-Type'
]))
.then(CheckTransactionIdHeaders)
.then(DoesNotHaveHeaders([
'X-Timestamp',
'Accept-Ranges',
'Access-Control-Allow-Origin',
'Access-Control-Expose-Headers',
'Date',
// Hmmm....
'Content-Range',
'X-Account-Bytes-Used',
'X-Account-Container-Count',
'X-Account-Object-Count',
'X-Container-Bytes-Used',
'X-Container-Object-Count'
]))
}
function MakeS3Request (service, operation, params) {
return new Promise((resolve, reject) => {
const s3req = service[operation](params)
// Don't *actually* send it
s3req.removeListener('send', AWS.EventListeners.Core.SEND)
// Instead, copy method, path, headers over to a new test-harness request
s3req.addListener('send', function () {
const endpoint = s3req.httpRequest.endpoint
const signedReq = s3req.httpRequest
const filteredHeaders = {}
for (const header of Object.keys(signedReq.headers)) {
if (header === 'Host' || header === 'Content-Length') {
continue // browser won't let you send these anyway
}
filteredHeaders[header] = signedReq.headers[header]
}
resolve(MakeRequest(
signedReq.method,
endpoint.protocol + '//' + endpoint.host + signedReq.path,
filteredHeaders,
signedReq.body
))
})
s3req.send()
})
}
function makeTests (params) {
const service = new AWS.S3(params)
return [
['presigned GET, no CORS',
() => MakeRequest('GET', service.getSignedUrl('getObject', {
Bucket: 'public-no-cors',
Key: 'obj'
}))
.then(CorsBlocked)],
['presigned HEAD, no CORS',
() => MakeRequest('HEAD', service.getSignedUrl('headObject', {
Bucket: 'public-no-cors',
Key: 'obj'
}))
.then(CorsBlocked)],
['presigned GET, object exists',
() => MakeRequest('GET', service.getSignedUrl('getObject', {
Bucket: 'private-with-cors',
Key: 'obj'
}))
.then(HasStatus(200, 'OK'))
.then(CheckS3Headers)
.then(HasHeaders(['x-amz-meta-mtime']))
.then(DoesNotHaveHeaders(['X-Object-Meta-Mtime']))
.then(HasHeaders({
'Content-Type': 'application/octet-stream',
Etag: '"0f343b0931126a20f133d67c2b018a3b"'
}))
.then(BodyHasLength(1024))],
['presigned HEAD, object exists',
() => MakeRequest('HEAD', service.getSignedUrl('headObject', {
Bucket: 'private-with-cors',
Key: 'obj'
}))
.then(HasStatus(200, 'OK'))
.then(CheckS3Headers)
.then(HasHeaders(['x-amz-meta-mtime']))
.then(DoesNotHaveHeaders(['X-Object-Meta-Mtime']))
.then(HasHeaders({
'Content-Type': 'application/octet-stream',
Etag: '"0f343b0931126a20f133d67c2b018a3b"'
}))
.then(HasNoBody)],
['GET, object exists',
() => MakeS3Request(service, 'getObject', {
Bucket: 'private-with-cors',
Key: 'obj'
})
.then(CorsBlocked)], // Pre-flight failed
['PUT',
() => MakeS3Request(service, 'putObject', {
Bucket: 'private-with-cors',
Key: 'put-target',
Body: 'test'
})
.then(CorsBlocked)], // Pre-flight failed
['GET If-Match matching',
() => MakeS3Request(service, 'getObject', {
Bucket: 'private-with-cors',
Key: 'obj',
IfMatch: '0f343b0931126a20f133d67c2b018a3b'
})
.then(CorsBlocked)], // Pre-flight failed
['GET Range',
() => MakeS3Request(service, 'getObject', {
Bucket: 'private-with-cors',
Key: 'obj',
Range: 'bytes=100-199'
})
.then(CorsBlocked)], // Pre-flight failed
]
}
runTests('s3 obj (v2)', makeTests({
endpoint: PARAMS.S3_ENDPOINT || 'http://localhost:8080',
region: PARAMS.S3_REGION || 'us-east-1',
accessKeyId: PARAMS.S3_USER || 'test:tester',
secretAccessKey: PARAMS.S3_KEY || 'testing',
s3ForcePathStyle: true,
signatureVersion: 'v2'
}))
runTests('s3 obj (v4)', makeTests({
endpoint: PARAMS.S3_ENDPOINT || 'http://localhost:8080',
region: PARAMS.S3_REGION || 'us-east-1',
accessKeyId: PARAMS.S3_USER || 'test:tester',
secretAccessKey: PARAMS.S3_KEY || 'testing',
s3ForcePathStyle: true,
signatureVersion: 'v4'
}))

10
test/cors/vendor/aws-sdk-2.829.0.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

73
test/unit/common/middleware/s3api/test_obj.py
View File

@ -1724,6 +1724,79 @@ class TestS3ApiObj(S3ApiTestCase):
'test:write', 'READ', src_path='')
self.assertEqual(status.split()[0], '400')
def test_cors_headers(self):
# note: Access-Control-Allow-Methods would normally be expected in
# response to an OPTIONS request but its included here in GET/PUT tests
# to check that it is always passed back in S3Response
cors_headers = {
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Methods': ('GET, PUT, POST, COPY, '
'DELETE, PUT, OPTIONS'),
'Access-Control-Expose-Headers':
'x-object-meta-test, x-object-meta-test=5funderscore, etag',
}
get_resp_headers = self.response_headers
get_resp_headers['x-object-meta-test=5funderscore'] = 'underscored'
self.swift.register(
'GET', '/v1/AUTH_test/bucket/cors-object', swob.HTTPOk,
dict(get_resp_headers, **cors_headers),
self.object_body)
self.swift.register(
'PUT', '/v1/AUTH_test/bucket/cors-object', swob.HTTPCreated,
dict({'etag': self.etag,
'last-modified': self.last_modified,
'x-object-meta-something': 'oh hai',
'x-object-meta-test=5funderscore': 'underscored'},
**cors_headers),
None)
req = Request.blank(
'/bucket/cors-object',
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header(),
'Origin': 'http://example.com',
'Access-Control-Request-Method': 'GET',
'Access-Control-Request-Headers': 'authorization'})
status, headers, body = self.call_s3api(req)
self.assertEqual(status, '200 OK')
self.assertIn('Access-Control-Allow-Origin', headers)
self.assertEqual(headers['Access-Control-Allow-Origin'], '*')
self.assertIn('Access-Control-Expose-Headers', headers)
self.assertEqual(
headers['Access-Control-Expose-Headers'],
'x-amz-meta-test, x-amz-meta-test_underscore, etag, '
'x-amz-request-id, x-amz-id-2')
self.assertIn('Access-Control-Allow-Methods', headers)
self.assertEqual(
headers['Access-Control-Allow-Methods'],
'GET, PUT, POST, DELETE, PUT, OPTIONS')
self.assertIn('x-amz-meta-test_underscore', headers)
self.assertEqual('underscored', headers['x-amz-meta-test_underscore'])
req = Request.blank(
'/bucket/cors-object',
environ={'REQUEST_METHOD': 'PUT'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header(),
'Origin': 'http://example.com',
'Access-Control-Request-Method': 'PUT',
'Access-Control-Request-Headers': 'authorization'})
status, headers, body = self.call_s3api(req)
self.assertEqual(status, '200 OK')
self.assertIn('Access-Control-Allow-Origin', headers)
self.assertEqual(headers['Access-Control-Allow-Origin'], '*')
self.assertIn('Access-Control-Expose-Headers', headers)
self.assertEqual(
headers['Access-Control-Expose-Headers'],
'x-amz-meta-test, x-amz-meta-test_underscore, etag, '
'x-amz-request-id, x-amz-id-2')
self.assertIn('Access-Control-Allow-Methods', headers)
self.assertEqual(
headers['Access-Control-Allow-Methods'],
'GET, PUT, POST, DELETE, PUT, OPTIONS')
self.assertEqual('underscored', headers['x-amz-meta-test_underscore'])
class TestS3ApiObjNonUTC(TestS3ApiObj):
def setUp(self):

2
test/unit/common/middleware/s3api/test_s3response.py
View File

@ -39,6 +39,7 @@ class TestResponse(unittest.TestCase):
resp = Response(headers={
'X-Object-Meta-Foo': 'Bar',
'X-Object-Meta-Non-\xdcnicode-Value': '\xff',
'X-Object-Meta-With=5FUnderscore': 'underscored',
'X-Object-Sysmeta-Baz': 'quux',
'Etag': 'unquoted',
'Content-type': 'text/plain',
@ -48,6 +49,7 @@ class TestResponse(unittest.TestCase):
self.assertEqual(dict(s3resp.headers), {
'x-amz-meta-foo': 'Bar',
'x-amz-meta-non-\xdcnicode-value': '\xff',
'x-amz-meta-with_underscore': 'underscored',
'ETag': '"unquoted"',
'Content-Type': 'text/plain',
'Content-Length': '0',

8
tools/playbooks/saio_single_node_setup/add_s3api.yaml
View File

@ -21,3 +21,11 @@
regexp: "container_sync tempauth"
replace: "container_sync s3api tempauth"
become: true
- name: Set s3_acl option
ini_file:
path: "/etc/swift/proxy-server.conf"
section: "filter:s3api"
option: "s3_acl"
value: "{{ s3_acl }}"
become: true