Enable override of container update headers

Crypto middleware needs to arrange for alternative
values of etag and content-type to be sent to container
servers with updates, since these will be encrypted with
a different key than the etag and content-type stored on
the object server.

Erasure coding apparently needs a similar capability.

This patch modifies the object server to overwrite the etag
and content-type values in the container update headers with
values that may optionally be specified by middleware in
X-Backend-Container-Update-Override-* headers.

Using the X-Backend- prefix ensures that these headers
cannot be sent or seen by clients.

A new probe test verifies the propagation of override
values from an internal client through the proxy, to
object server, to container server and then returned
in a container listing.

Change-Id: I7d846ed54ff173d08c66c6d5b0ecf7dff27f5a87

swift/obj/server.py

@@ -493,13 +493,20 @@ class ObjectController(BaseStorageServer):
                 self.delete_at_update(
                     'DELETE', orig_delete_at, account, container, obj,
                     request, device, policy_idx)
+        update_headers = HeaderKeyDict({
+            'x-size': metadata['Content-Length'],
+            'x-content-type': metadata['Content-Type'],
+            'x-timestamp': metadata['X-Timestamp'],
+            'x-etag': metadata['ETag']})
+        # apply any container update header overrides sent with request
+        for key, val in request.headers.iteritems():
+            override_prefix = 'x-backend-container-update-override-'
+            if key.lower().startswith(override_prefix):
+                override = key.lower().replace(override_prefix, 'x-')
+                update_headers[override] = val
         self.container_update(
             'PUT', account, container, obj, request,
-            HeaderKeyDict({
-                'x-size': metadata['Content-Length'],
-                'x-content-type': metadata['Content-Type'],
-                'x-timestamp': metadata['X-Timestamp'],
-                'x-etag': metadata['ETag']}),
+            update_headers,
             device, policy_idx)
         return HTTPCreated(request=request, etag=etag)
 

test/probe/test_object_async_update.py

@@ -14,12 +14,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
+import shutil
+
+from io import StringIO
+from tempfile import mkdtemp
+from textwrap import dedent
 from unittest import main
 from uuid import uuid4
 
 from swiftclient import client
 
-from swift.common import direct_client
+from swift.common import direct_client, internal_client
 from swift.common.manager import Manager
 from test.probe.common import kill_nonprimary_server, \
     kill_server, ReplProbeTest, start_server
@@ -58,5 +64,72 @@ class TestObjectAsyncUpdate(ReplProbeTest):
         self.assert_(obj in objs)
 
 
+class TestUpdateOverrides(ReplProbeTest):
+    """
+    Use an internal client to PUT an object to proxy server,
+    bypassing gatekeeper so that X-Backend- headers can be included.
+    Verify that the update override headers take effect and override
+    values propagate to the container server.
+    """
+    def setUp(self):
+        """
+        Reset all environment and start all servers.
+        """
+        super(TestUpdateOverrides, self).setUp()
+        self.tempdir = mkdtemp()
+        conf_path = os.path.join(self.tempdir, 'internal_client.conf')
+        conf_body = """
+        [DEFAULT]
+        swift_dir = /etc/swift
+
+        [pipeline:main]
+        pipeline = catch_errors cache proxy-server
+
+        [app:proxy-server]
+        use = egg:swift#proxy
+
+        [filter:cache]
+        use = egg:swift#memcache
+
+        [filter:catch_errors]
+        use = egg:swift#catch_errors
+        """
+        with open(conf_path, 'w') as f:
+            f.write(dedent(conf_body))
+        self.int_client = internal_client.InternalClient(conf_path, 'test', 1)
+
+    def tearDown(self):
+        super(TestUpdateOverrides, self).tearDown()
+        shutil.rmtree(self.tempdir)
+
+    def test(self):
+        headers = {
+            'Content-Type': 'text/plain',
+            'X-Backend-Container-Update-Override-Etag': 'override-etag',
+            'X-Backend-Container-Update-Override-Content-Type': 'override-type'
+        }
+        client.put_container(self.url, self.token, 'c1')
+
+        self.int_client.upload_object(StringIO(u'stuff'), self.account,
+                                      'c1', 'o1', headers)
+
+        # Run the object-updaters to be sure updates are done
+        Manager(['object-updater']).once()
+
+        meta = self.int_client.get_object_metadata(self.account, 'c1', 'o1')
+
+        self.assertEqual('text/plain', meta['content-type'])
+        self.assertEqual('c13d88cb4cb02003daedb8a84e5d272a', meta['etag'])
+
+        obj_iter = self.int_client.iter_objects(self.account, 'c1')
+        for obj in obj_iter:
+            if obj['name'] == 'o1':
+                self.assertEqual('override-etag', obj['hash'])
+                self.assertEqual('override-type', obj['content_type'])
+                break
+        else:
+            self.fail('Failed to find object o1 in listing')
+
+
 if __name__ == '__main__':
     main()

test/unit/obj/test_server.py

@@ -3056,6 +3056,48 @@ class TestObjectController(unittest.TestCase):
             'x-trans-id': '123',
             'referer': 'PUT http://localhost/sda1/0/a/c/o'}))
 
+    def test_container_update_overrides(self):
+        container_updates = []
+
+        def capture_updates(ip, port, method, path, headers, *args, **kwargs):
+            container_updates.append((ip, port, method, path, headers))
+
+        headers = {
+            'X-Timestamp': 1,
+            'X-Trans-Id': '123',
+            'X-Container-Host': 'chost:cport',
+            'X-Container-Partition': 'cpartition',
+            'X-Container-Device': 'cdevice',
+            'Content-Type': 'text/plain',
+            'X-Backend-Container-Update-Override-Etag': 'override_etag',
+            'X-Backend-Container-Update-Override-Content-Type': 'override_val',
+            'X-Backend-Container-Update-Override-Foo': 'bar',
+            'X-Backend-Container-Ignored': 'ignored'
+        }
+        req = Request.blank('/sda1/0/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
+                            headers=headers, body='')
+        with mocked_http_conn(
+                200, give_connect=capture_updates) as fake_conn:
+            resp = req.get_response(self.object_controller)
+            self.assertRaises(StopIteration, fake_conn.code_iter.next)
+        self.assertEqual(resp.status_int, 201)
+        self.assertEqual(len(container_updates), 1)
+        ip, port, method, path, headers = container_updates[0]
+        self.assertEqual(ip, 'chost')
+        self.assertEqual(port, 'cport')
+        self.assertEqual(method, 'PUT')
+        self.assertEqual(path, '/cdevice/cpartition/a/c/o')
+        self.assertEqual(headers, HeaderKeyDict({
+            'user-agent': 'object-server %s' % os.getpid(),
+            'x-size': '0',
+            'x-etag': 'override_etag',
+            'x-content-type': 'override_val',
+            'x-timestamp': utils.Timestamp(1).internal,
+            'X-Backend-Storage-Policy-Index': '0',  # default when not given
+            'x-trans-id': '123',
+            'referer': 'PUT http://localhost/sda1/0/a/c/o',
+            'x-foo': 'bar'}))
+
     def test_container_update_async(self):
         req = Request.blank(
             '/sda1/0/a/c/o',