From f9d856fc263908fa6116d89bfbe1d2149945fab1 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Fri, 4 Sep 2015 01:12:38 +0000 Subject: [PATCH] Transcribe more headers for responses Per AWS's docs, Cache-Control and Expires may be set on upload [1]. On download, the same headers would then be included in the response. Previously, these would not be included in Swift3 responses; now they will be. Additionally, several headers may be set on download via query parameters. This functionality already exists, but AWS's docs specify that this is "a subset of the headers that Amazon S3 accepts when you create an object" [2], so we should ensure Content-Language and Content-Disposition are transcribed as well. Finally, there is at least one undocumented header, X-Robots-Tag, which AWS allows to be set. At the very least, Boto [3] knows to try. Note that setting all of these headers already worked in Swift3, but requires that you update the allowed_headers option in the [app:object-server] section of object-server.conf. The conf used for functional tests has been so updated. [1] http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html#RESTObjectPUT-requests-request-headers [2] http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html#RESTObjectGET-requests-request-parameters [3] https://github.com/boto/boto/commit/0c11983 Change-Id: I22001c6fd14033a9f13c36a3e05fdc678c75654f --- etc/object-server.conf-sample | 18 ++++++++++ swift3/response.py | 4 ++- .../functional/conf/object-server.conf.in | 3 ++ swift3/test/functional/test_object.py | 36 ++++++++++++++++--- swift3/test/unit/test_obj.py | 28 +++++++++++---- 5 files changed, 77 insertions(+), 12 deletions(-) create mode 100644 etc/object-server.conf-sample diff --git a/etc/object-server.conf-sample b/etc/object-server.conf-sample new file mode 100644 index 00000000..f8402987 --- /dev/null +++ b/etc/object-server.conf-sample @@ -0,0 +1,18 @@ +[DEFAULT] + +[pipeline:main] +# This is the minimum pipeline for Swift (and Swift3) +pipeline = object-server + +[app:object-server] +use = egg:swift#object +# Comma separated list of headers that can be set in metadata on an object. +# This list is in addition to X-Object-Meta-* headers and cannot include +# Content-Type, etag, Content-Length, or deleted +# +# Note that S3 allows more headers than the default Swift object-server +# configuration. In particular, You may need to add Cache-Control, +# Content-Language, Expires, and X-Robots-Tag +allowed_headers = Cache-Control, Content-Disposition, Content-Encoding, + Content-Language, Expires, X-Delete-At, X-Object-Manifest, X-Robots-Tag, + X-Static-Large-Object diff --git a/swift3/response.py b/swift3/response.py index ee44fd13..3a069c79 100644 --- a/swift3/response.py +++ b/swift3/response.py @@ -102,7 +102,9 @@ class Response(ResponseBase, swob.Response): headers['x-amz-meta-' + _key[14:]] = val elif _key in ('content-length', 'content-type', 'content-range', 'content-encoding', - 'etag', 'last-modified'): + 'content-disposition', 'content-language', + 'etag', 'last-modified', 'x-robots-tag', + 'cache-control', 'expires'): headers[key] = val elif _key == 'x-static-large-object': # for delete slo diff --git a/swift3/test/functional/conf/object-server.conf.in b/swift3/test/functional/conf/object-server.conf.in index bf156efe..9cf7dd58 100644 --- a/swift3/test/functional/conf/object-server.conf.in +++ b/swift3/test/functional/conf/object-server.conf.in @@ -12,3 +12,6 @@ pipeline = object-server [app:object-server] use = egg:swift#object +allowed_headers = Cache-Control, Content-Disposition, Content-Encoding, + Content-Language, Expires, X-Delete-At, X-Object-Manifest, X-Robots-Tag, + X-Static-Large-Object diff --git a/swift3/test/functional/test_object.py b/swift3/test/functional/test_object.py index 31069b99..f366cb76 100644 --- a/swift3/test/functional/test_object.py +++ b/swift3/test/functional/test_object.py @@ -278,21 +278,47 @@ class TestSwift3Object(Swift3FunctionalTestCase): self.assertCommonResponseHeaders(headers) self._assertObjectEtag(self.bucket, obj, etag) - def test_put_object_metadata(self): + def _test_put_object_headers(self, req_headers): obj = 'object' content = 'abcdefghij' etag = md5(content).hexdigest() - headers = {'X-Amz-Meta-Bar': 'foo', 'X-Amz-Meta-Bar2': 'foo2'} status, headers, body = \ - self.conn.make_request('PUT', self.bucket, obj, headers, content) + self.conn.make_request('PUT', self.bucket, obj, + req_headers, content) self.assertEquals(status, 200) status, headers, body = \ self.conn.make_request('HEAD', self.bucket, obj) - self.assertEquals(headers['x-amz-meta-bar'], 'foo') - self.assertEquals(headers['x-amz-meta-bar2'], 'foo2') + for header, value in req_headers.items(): + self.assertIn(header.lower(), headers) + self.assertEquals(headers[header.lower()], value) self.assertCommonResponseHeaders(headers) self._assertObjectEtag(self.bucket, obj, etag) + def test_put_object_metadata(self): + self._test_put_object_headers({ + 'X-Amz-Meta-Bar': 'foo', + 'X-Amz-Meta-Bar2': 'foo2'}) + + def test_put_object_content_headers(self): + self._test_put_object_headers({ + 'Content-Type': 'foo/bar', + 'Content-Encoding': 'baz', + 'Content-Disposition': 'attachment', + 'Content-Language': 'en'}) + + def test_put_object_cache_control(self): + self._test_put_object_headers({ + 'Cache-Control': 'private, some-extension'}) + + def test_put_object_expires(self): + self._test_put_object_headers({ + # We don't validate that the Expires header is a valid date + 'Expires': 'a valid HTTP-date timestamp'}) + + def test_put_object_robots_tag(self): + self._test_put_object_headers({ + 'X-Robots-Tag': 'googlebot: noarchive'}) + def test_put_object_storage_class(self): obj = 'object' content = 'abcdefghij' diff --git a/swift3/test/unit/test_obj.py b/swift3/test/unit/test_obj.py index ca6342d9..821b2cb1 100644 --- a/swift3/test/unit/test_obj.py +++ b/swift3/test/unit/test_obj.py @@ -60,9 +60,14 @@ class TestSwift3Obj(Swift3TestCase): self.response_headers = {'Content-Type': 'text/html', 'Content-Length': len(self.object_body), + 'Content-Disposition': 'inline', + 'Content-Language': 'en', 'x-object-meta-test': 'swift', 'etag': self.etag, - 'last-modified': self.last_modified} + 'last-modified': self.last_modified, + 'expires': 'Mon, 21 Sep 2015 12:00:00 GMT', + 'x-robots-tag': 'nofollow', + 'cache-control': 'private'} self.swift.register('GET', '/v1/AUTH_test/bucket/object', swob.HTTPOk, self.response_headers, @@ -80,16 +85,27 @@ class TestSwift3Obj(Swift3TestCase): status, headers, body = self.call_swift3(req) self.assertEquals(status.split()[0], '200') + unexpected_headers = [] for key, val in self.response_headers.iteritems(): - if key in ('content-length', 'content-type', 'content-encoding', - 'last-modified'): - self.assertTrue(key in headers) - self.assertEquals(headers[key], val) + if key in ('Content-Length', 'Content-Type', 'content-encoding', + 'last-modified', 'cache-control', 'Content-Disposition', + 'Content-Language', 'expires', 'x-robots-tag'): + self.assertIn(key, headers) + self.assertEquals(headers[key], str(val)) + + elif key == 'etag': + self.assertEquals(headers[key], '"%s"' % val) elif key.startswith('x-object-meta-'): - self.assertTrue('x-amz-meta-' + key[14:] in headers) + self.assertIn('x-amz-meta-' + key[14:], headers) self.assertEquals(headers['x-amz-meta-' + key[14:]], val) + else: + unexpected_headers.append((key, val)) + + if unexpected_headers: + self.fail('unexpected headers: %r' % unexpected_headers) + self.assertEquals(headers['etag'], '"%s"' % self.response_headers['etag'])