Transcribe more headers for responses

Per AWS's docs, Cache-Control and Expires may be set on upload [1]. On
download, the same headers would then be included in the response.
Previously, these would not be included in Swift3 responses; now they
will be.

Additionally, several headers may be set on download via query
parameters. This functionality already exists, but AWS's docs specify
that this is "a subset of the headers that Amazon S3 accepts when you
create an object" [2], so we should ensure Content-Language and
Content-Disposition are transcribed as well.

Finally, there is at least one undocumented header, X-Robots-Tag, which
AWS allows to be set. At the very least, Boto [3] knows to try.

Note that setting all of these headers already worked in Swift3, but
requires that you update the allowed_headers option in the
[app:object-server] section of object-server.conf. The conf used for
functional tests has been so updated.

[1] http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html#RESTObjectPUT-requests-request-headers

[2] http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html#RESTObjectGET-requests-request-parameters

[3] https://github.com/boto/boto/commit/0c11983

Change-Id: I22001c6fd14033a9f13c36a3e05fdc678c75654f

etc/object-server.conf-sample

@@ -0,0 +1,18 @@
+[DEFAULT]
+
+[pipeline:main]
+# This is the minimum pipeline for Swift (and Swift3)
+pipeline = object-server
+
+[app:object-server]
+use = egg:swift#object
+# Comma separated list of headers that can be set in metadata on an object.
+# This list is in addition to X-Object-Meta-* headers and cannot include
+# Content-Type, etag, Content-Length, or deleted
+#
+# Note that S3 allows more headers than the default Swift object-server
+# configuration. In particular, You may need to add Cache-Control,
+# Content-Language, Expires, and X-Robots-Tag
+allowed_headers = Cache-Control, Content-Disposition, Content-Encoding,
+ Content-Language, Expires, X-Delete-At, X-Object-Manifest, X-Robots-Tag,
+ X-Static-Large-Object

swift3/response.py

@@ -102,7 +102,9 @@ class Response(ResponseBase, swob.Response):
                 headers['x-amz-meta-' + _key[14:]] = val
             elif _key in ('content-length', 'content-type',
                           'content-range', 'content-encoding',
-                          'etag', 'last-modified'):
+                          'content-disposition', 'content-language',
+                          'etag', 'last-modified', 'x-robots-tag',
+                          'cache-control', 'expires'):
                 headers[key] = val
             elif _key == 'x-static-large-object':
                 # for delete slo

swift3/test/functional/conf/object-server.conf.in

@@ -12,3 +12,6 @@ pipeline = object-server
 
 [app:object-server]
 use = egg:swift#object
+allowed_headers = Cache-Control, Content-Disposition, Content-Encoding,
+ Content-Language, Expires, X-Delete-At, X-Object-Manifest, X-Robots-Tag,
+ X-Static-Large-Object

swift3/test/functional/test_object.py

@@ -278,21 +278,47 @@ class TestSwift3Object(Swift3FunctionalTestCase):
         self.assertCommonResponseHeaders(headers)
         self._assertObjectEtag(self.bucket, obj, etag)
 
-    def test_put_object_metadata(self):
+    def _test_put_object_headers(self, req_headers):
         obj = 'object'
         content = 'abcdefghij'
         etag = md5(content).hexdigest()
-        headers = {'X-Amz-Meta-Bar': 'foo', 'X-Amz-Meta-Bar2': 'foo2'}
         status, headers, body = \
-            self.conn.make_request('PUT', self.bucket, obj, headers, content)
+            self.conn.make_request('PUT', self.bucket, obj,
+                                   req_headers, content)
         self.assertEquals(status, 200)
         status, headers, body = \
             self.conn.make_request('HEAD', self.bucket, obj)
-        self.assertEquals(headers['x-amz-meta-bar'], 'foo')
-        self.assertEquals(headers['x-amz-meta-bar2'], 'foo2')
+        for header, value in req_headers.items():
+            self.assertIn(header.lower(), headers)
+            self.assertEquals(headers[header.lower()], value)
         self.assertCommonResponseHeaders(headers)
         self._assertObjectEtag(self.bucket, obj, etag)
 
+    def test_put_object_metadata(self):
+        self._test_put_object_headers({
+            'X-Amz-Meta-Bar': 'foo',
+            'X-Amz-Meta-Bar2': 'foo2'})
+
+    def test_put_object_content_headers(self):
+        self._test_put_object_headers({
+            'Content-Type': 'foo/bar',
+            'Content-Encoding': 'baz',
+            'Content-Disposition': 'attachment',
+            'Content-Language': 'en'})
+
+    def test_put_object_cache_control(self):
+        self._test_put_object_headers({
+            'Cache-Control': 'private, some-extension'})
+
+    def test_put_object_expires(self):
+        self._test_put_object_headers({
+            # We don't validate that the Expires header is a valid date
+            'Expires': 'a valid HTTP-date timestamp'})
+
+    def test_put_object_robots_tag(self):
+        self._test_put_object_headers({
+            'X-Robots-Tag': 'googlebot: noarchive'})
+
     def test_put_object_storage_class(self):
         obj = 'object'
         content = 'abcdefghij'

swift3/test/unit/test_obj.py

@@ -60,9 +60,14 @@ class TestSwift3Obj(Swift3TestCase):
 
         self.response_headers = {'Content-Type': 'text/html',
                                  'Content-Length': len(self.object_body),
+                                 'Content-Disposition': 'inline',
+                                 'Content-Language': 'en',
                                  'x-object-meta-test': 'swift',
                                  'etag': self.etag,
-                                 'last-modified': self.last_modified}
+                                 'last-modified': self.last_modified,
+                                 'expires': 'Mon, 21 Sep 2015 12:00:00 GMT',
+                                 'x-robots-tag': 'nofollow',
+                                 'cache-control': 'private'}
 
         self.swift.register('GET', '/v1/AUTH_test/bucket/object',
                             swob.HTTPOk, self.response_headers,
@@ -80,16 +85,27 @@ class TestSwift3Obj(Swift3TestCase):
         status, headers, body = self.call_swift3(req)
         self.assertEquals(status.split()[0], '200')
 
+        unexpected_headers = []
         for key, val in self.response_headers.iteritems():
-            if key in ('content-length', 'content-type', 'content-encoding',
-                       'last-modified'):
-                self.assertTrue(key in headers)
-                self.assertEquals(headers[key], val)
+            if key in ('Content-Length', 'Content-Type', 'content-encoding',
+                       'last-modified', 'cache-control', 'Content-Disposition',
+                       'Content-Language', 'expires', 'x-robots-tag'):
+                self.assertIn(key, headers)
+                self.assertEquals(headers[key], str(val))
+
+            elif key == 'etag':
+                self.assertEquals(headers[key], '"%s"' % val)
 
             elif key.startswith('x-object-meta-'):
-                self.assertTrue('x-amz-meta-' + key[14:] in headers)
+                self.assertIn('x-amz-meta-' + key[14:], headers)
                 self.assertEquals(headers['x-amz-meta-' + key[14:]], val)
 
+            else:
+                unexpected_headers.append((key, val))
+
+        if unexpected_headers:
+                self.fail('unexpected headers: %r' % unexpected_headers)
+
         self.assertEquals(headers['etag'],
                           '"%s"' % self.response_headers['etag'])