From 4d4885acdc8107c4f8f8cd10bc6d19ba3901eb25 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Fri, 5 Aug 2016 18:05:45 +0000 Subject: [PATCH] Tighten header checks for object PUT/POST paths Change-Id: If2cd059719fe5af1e73ecde5306e9f68d590831f --- test/unit/obj/test_server.py | 193 ++++++++++++++++-------- test/unit/proxy/controllers/test_obj.py | 78 +++++++++- 2 files changed, 204 insertions(+), 67 deletions(-) diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index 5262c6d876..40e0852a4b 100755 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -181,79 +181,104 @@ class TestObjectController(unittest.TestCase): original_headers = self.object_controller.allowed_headers test_headers = 'content-encoding foo bar'.split() self.object_controller.allowed_headers = set(test_headers) - timestamp = normalize_timestamp(time()) + put_timestamp = normalize_timestamp(time()) + headers = {'X-Timestamp': put_timestamp, + 'Content-Type': 'application/x-test', + 'Foo': 'fooheader', + 'Baz': 'bazheader', + 'X-Object-Sysmeta-Color': 'blue', + 'X-Object-Meta-1': 'One', + 'X-Object-Meta-Two': 'Two'} req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': timestamp, - 'Content-Type': 'application/x-test', - 'Foo': 'fooheader', - 'Baz': 'bazheader', - 'X-Object-Meta-1': 'One', - 'X-Object-Meta-Two': 'Two'}) + headers=headers) req.body = 'VERIFY' + etag = '"%s"' % md5('VERIFY').hexdigest() resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 201) + self.assertEqual(dict(resp.headers), { + 'Content-Type': 'text/html; charset=UTF-8', + 'Content-Length': str(len(resp.body)), + 'Etag': etag, + }) - timestamp = normalize_timestamp(time()) + post_timestamp = normalize_timestamp(time()) + headers = {'X-Timestamp': post_timestamp, + 'X-Object-Meta-3': 'Three', + 'X-Object-Meta-4': 'Four', + 'Content-Encoding': 'gzip', + 'Foo': 'fooheader', + 'Bar': 'barheader', + 'Content-Type': 'application/x-test'} req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Timestamp': timestamp, - 'X-Object-Meta-3': 'Three', - 'X-Object-Meta-4': 'Four', - 'Content-Encoding': 'gzip', - 'Foo': 'fooheader', - 'Bar': 'barheader', - 'Content-Type': 'application/x-test'}) + headers=headers) resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 202) + self.assertEqual(dict(resp.headers), { + 'Content-Type': 'text/html; charset=UTF-8', + 'Content-Length': str(len(resp.body)), + }) req = Request.blank('/sda1/p/a/c/o') resp = req.get_response(self.object_controller) - self.assertNotIn("X-Object-Meta-1", resp.headers) - self.assertNotIn("X-Object-Meta-Two", resp.headers) - self.assertIn("X-Object-Meta-3", resp.headers) - self.assertIn("X-Object-Meta-4", resp.headers) - self.assertIn("Foo", resp.headers) - self.assertIn("Bar", resp.headers) - self.assertNotIn("Baz", resp.headers) - self.assertIn("Content-Encoding", resp.headers) - self.assertEqual(resp.headers['Content-Type'], 'application/x-test') + expected_headers = { + 'Content-Type': 'application/x-test', + 'Content-Length': '6', + 'Etag': etag, + 'X-Object-Sysmeta-Color': 'blue', + 'X-Object-Meta-3': 'Three', + 'X-Object-Meta-4': 'Four', + 'Foo': 'fooheader', + 'Bar': 'barheader', + 'Content-Encoding': 'gzip', + 'X-Backend-Timestamp': post_timestamp, + 'X-Timestamp': post_timestamp, + 'Last-Modified': strftime( + '%a, %d %b %Y %H:%M:%S GMT', + gmtime(math.ceil(float(post_timestamp)))), + } + self.assertEqual(dict(resp.headers), expected_headers) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.object_controller) - self.assertNotIn("X-Object-Meta-1", resp.headers) - self.assertNotIn("X-Object-Meta-Two", resp.headers) - self.assertIn("X-Object-Meta-3", resp.headers) - self.assertIn("X-Object-Meta-4", resp.headers) - self.assertIn("Foo", resp.headers) - self.assertIn("Bar", resp.headers) - self.assertNotIn("Baz", resp.headers) - self.assertIn("Content-Encoding", resp.headers) - self.assertEqual(resp.headers['Content-Type'], 'application/x-test') + self.assertEqual(dict(resp.headers), expected_headers) - timestamp = normalize_timestamp(time()) + post_timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Timestamp': timestamp, + headers={'X-Timestamp': post_timestamp, + 'X-Object-Sysmeta-Color': 'red', 'Content-Type': 'application/x-test'}) resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 202) + self.assertEqual(dict(resp.headers), { + 'Content-Type': 'text/html; charset=UTF-8', + 'Content-Length': str(len(resp.body)), + }) + req = Request.blank('/sda1/p/a/c/o') resp = req.get_response(self.object_controller) - self.assertNotIn("X-Object-Meta-3", resp.headers) - self.assertNotIn("X-Object-Meta-4", resp.headers) - self.assertNotIn("Foo", resp.headers) - self.assertNotIn("Bar", resp.headers) - self.assertNotIn("Content-Encoding", resp.headers) - self.assertEqual(resp.headers['Content-Type'], 'application/x-test') + self.assertEqual(dict(resp.headers), { + 'Content-Type': 'application/x-test', + 'Content-Length': '6', + 'Etag': etag, + 'X-Object-Sysmeta-Color': 'blue', + 'X-Backend-Timestamp': post_timestamp, + 'X-Timestamp': post_timestamp, + 'Last-Modified': strftime( + '%a, %d %b %Y %H:%M:%S GMT', + gmtime(math.ceil(float(post_timestamp)))), + }) # test defaults self.object_controller.allowed_headers = original_headers - timestamp = normalize_timestamp(time()) + put_timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': timestamp, + headers={'X-Timestamp': put_timestamp, 'Content-Type': 'application/x-test', 'Foo': 'fooheader', + 'X-Object-Sysmeta-Color': 'red', 'X-Object-Meta-1': 'One', 'X-Object-Manifest': 'c/bar', 'Content-Encoding': 'gzip', @@ -263,48 +288,90 @@ class TestObjectController(unittest.TestCase): req.body = 'VERIFY' resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 201) + self.assertEqual(dict(resp.headers), { + 'Content-Type': 'text/html; charset=UTF-8', + 'Content-Length': str(len(resp.body)), + 'Etag': etag, + }) + req = Request.blank('/sda1/p/a/c/o') resp = req.get_response(self.object_controller) - self.assertIn("X-Object-Meta-1", resp.headers) - self.assertNotIn("Foo", resp.headers) - self.assertIn("Content-Encoding", resp.headers) - self.assertIn("X-Object-Manifest", resp.headers) - self.assertIn("Content-Disposition", resp.headers) - self.assertIn("X-Static-Large-Object", resp.headers) - self.assertEqual(resp.headers['Content-Type'], 'application/x-test') + self.assertEqual(dict(resp.headers), { + 'Content-Type': 'application/x-test', + 'Content-Length': '6', + 'Etag': etag, + 'X-Object-Sysmeta-Color': 'red', + 'X-Object-Meta-1': 'One', + 'Content-Encoding': 'gzip', + 'X-Object-Manifest': 'c/bar', + 'Content-Disposition': 'bar', + 'X-Static-Large-Object': 'True', + 'X-Backend-Timestamp': put_timestamp, + 'X-Timestamp': put_timestamp, + 'Last-Modified': strftime( + '%a, %d %b %Y %H:%M:%S GMT', + gmtime(math.ceil(float(put_timestamp)))), + }) - timestamp = normalize_timestamp(time()) + post_timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Timestamp': timestamp, + headers={'X-Timestamp': post_timestamp, 'X-Object-Meta-3': 'Three', 'Foo': 'fooheader', 'Content-Type': 'application/x-test'}) resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 202) + self.assertEqual(dict(resp.headers), { + 'Content-Type': 'text/html; charset=UTF-8', + 'Content-Length': str(len(resp.body)), + }) + req = Request.blank('/sda1/p/a/c/o') resp = req.get_response(self.object_controller) - self.assertNotIn("X-Object-Meta-1", resp.headers) - self.assertNotIn("Foo", resp.headers) - self.assertNotIn("Content-Encoding", resp.headers) - self.assertNotIn("X-Object-Manifest", resp.headers) - self.assertNotIn("Content-Disposition", resp.headers) - self.assertIn("X-Object-Meta-3", resp.headers) - self.assertIn("X-Static-Large-Object", resp.headers) - self.assertEqual(resp.headers['Content-Type'], 'application/x-test') + self.assertEqual(dict(resp.headers), { + 'Content-Type': 'application/x-test', + 'Content-Length': '6', + 'Etag': etag, + 'X-Object-Sysmeta-Color': 'red', + 'X-Object-Meta-3': 'Three', + 'X-Static-Large-Object': 'True', + 'X-Backend-Timestamp': post_timestamp, + 'X-Timestamp': post_timestamp, + 'Last-Modified': strftime( + '%a, %d %b %Y %H:%M:%S GMT', + gmtime(math.ceil(float(post_timestamp)))), + }) # Test for empty metadata - timestamp = normalize_timestamp(time()) + post_timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Timestamp': timestamp, + headers={'X-Timestamp': post_timestamp, 'Content-Type': 'application/x-test', 'X-Object-Meta-3': ''}) resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 202) + self.assertEqual(dict(resp.headers), { + 'Content-Type': 'text/html; charset=UTF-8', + 'Content-Length': str(len(resp.body)), + }) + req = Request.blank('/sda1/p/a/c/o') resp = req.get_response(self.object_controller) - self.assertEqual(resp.headers["x-object-meta-3"], '') + self.assertEqual(dict(resp.headers), { + 'Content-Type': 'application/x-test', + 'Content-Length': '6', + 'Etag': etag, + 'X-Object-Sysmeta-Color': 'red', + 'X-Object-Meta-3': '', + 'X-Static-Large-Object': 'True', + 'X-Backend-Timestamp': post_timestamp, + 'X-Timestamp': post_timestamp, + 'Last-Modified': strftime( + '%a, %d %b %Y %H:%M:%S GMT', + gmtime(math.ceil(float(post_timestamp)))), + }) def test_POST_old_timestamp(self): ts = time() diff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py index d30525a985..bec610c5a6 100755 --- a/test/unit/proxy/controllers/test_obj.py +++ b/test/unit/proxy/controllers/test_obj.py @@ -16,6 +16,7 @@ import email.parser import itertools +import math import random import time import unittest @@ -626,12 +627,28 @@ class TestReplicatedObjController(BaseObjectControllerMixin, codes = [201] * self.replicas() expect_headers = {'X-Obj-Metadata-Footer': 'yes'} + resp_headers = { + 'Some-Header': 'Four', + 'Etag': '"%s"' % etag, + } with set_http_connect(*codes, expect_headers=expect_headers, give_send=capture_body, - give_connect=capture_headers): + give_connect=capture_headers, + headers=resp_headers): resp = req.get_response(self.app) self.assertEqual(resp.status_int, 201) + timestamps = {captured_req['headers']['x-timestamp'] + for captured_req in put_requests.values()} + self.assertEqual(1, len(timestamps), timestamps) + self.assertEqual(dict(resp.headers), { + 'Content-Type': 'text/html; charset=UTF-8', + 'Content-Length': '0', + 'Etag': etag, + 'Last-Modified': time.strftime( + "%a, %d %b %Y %H:%M:%S GMT", + time.gmtime(math.ceil(float(timestamps.pop())))), + }) for connection_id, info in put_requests.items(): body = ''.join(info['chunks']) headers = info['headers'] @@ -689,12 +706,29 @@ class TestReplicatedObjController(BaseObjectControllerMixin, conn_id = kwargs['connection_id'] put_requests[conn_id]['headers'] = headers + resp_headers = { + 'Etag': '"resp_etag"', + # NB: ignored! + 'Some-Header': 'Four', + } with set_http_connect(*codes, expect_headers=expect_headers, give_send=capture_body, - give_connect=capture_headers): + give_connect=capture_headers, + headers=resp_headers): resp = req.get_response(self.app) self.assertEqual(resp.status_int, 201) + timestamps = {captured_req['headers']['x-timestamp'] + for captured_req in put_requests.values()} + self.assertEqual(1, len(timestamps), timestamps) + self.assertEqual(dict(resp.headers), { + 'Content-Type': 'text/html; charset=UTF-8', + 'Content-Length': '0', + 'Etag': 'resp_etag', + 'Last-Modified': time.strftime( + "%a, %d %b %Y %H:%M:%S GMT", + time.gmtime(math.ceil(float(timestamps.pop())))), + }) for connection_id, info in put_requests.items(): body = unchunk_body(''.join(info['chunks'])) headers = info['headers'] @@ -1892,6 +1926,10 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): 'X-Obj-Metadata-Footer': 'yes', 'X-Obj-Multiphase-Commit': 'yes' } + resp_headers = { + 'Some-Other-Header': 'Four', + 'Etag': 'ignored', + } put_requests = defaultdict(lambda: {'boundary': None, 'chunks': []}) @@ -1905,13 +1943,27 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): 'X-Backend-Obj-Multipart-Mime-Boundary'] put_requests[conn_id]['backend-content-length'] = headers[ 'X-Backend-Obj-Content-Length'] + put_requests[conn_id]['x-timestamp'] = headers[ + 'X-Timestamp'] with set_http_connect(*codes, expect_headers=expect_headers, give_send=capture_body, - give_connect=capture_headers): + give_connect=capture_headers, + headers=resp_headers): resp = req.get_response(self.app) self.assertEqual(resp.status_int, 201) + timestamps = {captured_req['x-timestamp'] + for captured_req in put_requests.values()} + self.assertEqual(1, len(timestamps), timestamps) + self.assertEqual(dict(resp.headers), { + 'Content-Type': 'text/html; charset=UTF-8', + 'Content-Length': '0', + 'Last-Modified': time.strftime( + "%a, %d %b %Y %H:%M:%S GMT", + time.gmtime(math.ceil(float(timestamps.pop())))), + 'Etag': etag, + }) frag_archives = [] for connection_id, info in put_requests.items(): body = unchunk_body(''.join(info['chunks'])) @@ -2001,6 +2053,10 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): 'X-Obj-Metadata-Footer': 'yes', 'X-Obj-Multiphase-Commit': 'yes' } + resp_headers = { + 'Some-Other-Header': 'Four', + 'Etag': 'ignored', + } def do_test(footers_to_add, expect_added): put_requests = defaultdict( @@ -2014,6 +2070,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): conn_id = kwargs['connection_id'] put_requests[conn_id]['boundary'] = headers[ 'X-Backend-Obj-Multipart-Mime-Boundary'] + put_requests[conn_id]['x-timestamp'] = headers[ + 'X-Timestamp'] def footers_callback(footers): footers.update(footers_to_add) @@ -2023,10 +2081,22 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): with set_http_connect(*codes, expect_headers=expect_headers, give_send=capture_body, - give_connect=capture_headers): + give_connect=capture_headers, + headers=resp_headers): resp = req.get_response(self.app) self.assertEqual(resp.status_int, 201) + timestamps = {captured_req['x-timestamp'] + for captured_req in put_requests.values()} + self.assertEqual(1, len(timestamps), timestamps) + self.assertEqual(dict(resp.headers), { + 'Content-Type': 'text/html; charset=UTF-8', + 'Content-Length': '0', + 'Last-Modified': time.strftime( + "%a, %d %b %Y %H:%M:%S GMT", + time.gmtime(math.ceil(float(timestamps.pop())))), + 'Etag': etag, + }) for connection_id, info in put_requests.items(): body = unchunk_body(''.join(info['chunks'])) # email.parser.FeedParser doesn't know how to take a multipart