From 21adf82cf11fa80479676225a8a61c0029387cb5 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Fri, 8 Aug 2014 02:14:27 -0700 Subject: [PATCH] code shuffle post expired headers refactor Change-Id: I62248d7d3d7e0a3696a30e3d567ac6c2bea3c8eb --- swift/proxy/controllers/obj.py | 25 ++-- test/functional/test_object.py | 39 ++++++ test/unit/proxy/controllers/test_obj.py | 158 ++++++++++++++++++++++ test/unit/proxy/test_server.py | 170 ------------------------ 4 files changed, 207 insertions(+), 185 deletions(-) diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index b10e2dd18f..ec45c0739f 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -271,12 +271,8 @@ class ObjectController(Controller): if not containers: return HTTPNotFound(request=req) - try: - req, delete_at_container, delete_at_part, \ - delete_at_nodes = self._config_obj_expiration(req) - except ValueError as e: - return HTTPBadRequest(request=req, content_type='text/plain', - body=str(e)) + req, delete_at_container, delete_at_part, \ + delete_at_nodes = self._config_obj_expiration(req) # pass the policy index to storage nodes via req header policy_index = req.headers.get('X-Backend-Storage-Policy-Index', @@ -433,7 +429,8 @@ class ObjectController(Controller): try: x_delete_after = int(req.headers['x-delete-after']) except ValueError: - raise ValueError('Non-integer X-Delete-After') + raise HTTPBadRequest(request=req, content_type='text/plain', + body='Non-integer X-Delete-After') req.headers['x-delete-at'] = normalize_delete_at_timestamp( time.time() + x_delete_after) @@ -443,10 +440,12 @@ class ObjectController(Controller): x_delete_at = int(normalize_delete_at_timestamp( int(req.headers['x-delete-at']))) except ValueError: - raise ValueError('Non-integer X-Delete-At') + raise HTTPBadRequest(request=req, content_type='text/plain', + body='Non-integer X-Delete-At') if x_delete_at < time.time(): - raise ValueError('X-Delete-At in past') + raise HTTPBadRequest(request=req, content_type='text/plain', + body='X-Delete-At in past') req.environ.setdefault('swift.log_info', []).append( 'x-delete-at:%s' % x_delete_at) @@ -659,12 +658,8 @@ class ObjectController(Controller): req = sink_req - try: - req, delete_at_container, delete_at_part, \ - delete_at_nodes = self._config_obj_expiration(req) - except ValueError as e: - return HTTPBadRequest(request=req, content_type='text/plain', - body=str(e)) + req, delete_at_container, delete_at_part, \ + delete_at_nodes = self._config_obj_expiration(req) node_iter = GreenthreadSafeIterator( self.iter_nodes_local_first(obj_ring, partition)) diff --git a/test/functional/test_object.py b/test/functional/test_object.py index 6b29800515..0b3934c904 100755 --- a/test/functional/test_object.py +++ b/test/functional/test_object.py @@ -133,6 +133,45 @@ class TestObject(unittest.TestCase): resp.read() self.assertEquals(resp.status, 400) + def test_non_integer_x_delete_after(self): + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/%s' % (parsed.path, self.container, + 'non_integer_x_delete_after'), + '', {'X-Auth-Token': token, + 'Content-Length': '0', + 'X-Delete-After': '*'}) + return check_response(conn) + resp = retry(put) + body = resp.read() + self.assertEquals(resp.status, 400) + self.assertEqual(body, 'Non-integer X-Delete-After') + + def test_non_integer_x_delete_at(self): + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/%s' % (parsed.path, self.container, + 'non_integer_x_delete_at'), + '', {'X-Auth-Token': token, + 'Content-Length': '0', + 'X-Delete-At': '*'}) + return check_response(conn) + resp = retry(put) + body = resp.read() + self.assertEquals(resp.status, 400) + self.assertEqual(body, 'Non-integer X-Delete-At') + + def test_x_delete_at_in_the_past(self): + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/%s' % (parsed.path, self.container, + 'x_delete_at_in_the_past'), + '', {'X-Auth-Token': token, + 'Content-Length': '0', + 'X-Delete-At': '0'}) + return check_response(conn) + resp = retry(put) + body = resp.read() + self.assertEquals(resp.status, 400) + self.assertEqual(body, 'X-Delete-At in past') + def test_copy_object(self): if tf.skip: raise SkipTest diff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py index 645b2742d1..ac7313502a 100755 --- a/test/unit/proxy/controllers/test_obj.py +++ b/test/unit/proxy/controllers/test_obj.py @@ -249,6 +249,164 @@ class TestObjController(unittest.TestCase): resp = req.get_response(self.app) self.assertEquals(resp.status_int, 202) + def test_POST_delete_at(self): + t = str(int(time.time() + 100)) + req = swob.Request.blank('/v1/a/c/o', method='POST', + headers={'Content-Type': 'foo/bar', + 'X-Delete-At': t}) + post_headers = [] + + def capture_headers(ip, port, device, part, method, path, headers, + **kwargs): + if method == 'POST': + post_headers.append(headers) + x_newest_responses = [200] * self.obj_ring.replicas + \ + [404] * self.obj_ring.max_more_nodes + post_resp = [200] * self.obj_ring.replicas + codes = x_newest_responses + post_resp + with set_http_connect(*codes, give_connect=capture_headers): + resp = req.get_response(self.app) + self.assertEquals(resp.status_int, 200) + for given_headers in post_headers: + self.assertEquals(given_headers.get('X-Delete-At'), t) + self.assertTrue('X-Delete-At-Host' in given_headers) + self.assertTrue('X-Delete-At-Device' in given_headers) + self.assertTrue('X-Delete-At-Partition' in given_headers) + self.assertTrue('X-Delete-At-Container' in given_headers) + + def test_POST_non_int_delete_after(self): + t = str(int(time.time() + 100)) + '.1' + req = swob.Request.blank('/v1/a/c/o', method='POST', + headers={'Content-Type': 'foo/bar', + 'X-Delete-After': t}) + x_newest_responses = [200] * self.obj_ring.replicas + \ + [404] * self.obj_ring.max_more_nodes + with set_http_connect(*x_newest_responses): + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 400) + self.assertEqual('Non-integer X-Delete-After', resp.body) + + def test_POST_negative_delete_after(self): + req = swob.Request.blank('/v1/a/c/o', method='POST', + headers={'Content-Type': 'foo/bar', + 'X-Delete-After': '-60'}) + x_newest_responses = [200] * self.obj_ring.replicas + \ + [404] * self.obj_ring.max_more_nodes + with set_http_connect(*x_newest_responses): + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 400) + self.assertEqual('X-Delete-At in past', resp.body) + + def test_POST_delete_at_non_integer(self): + t = str(int(time.time() + 100)) + '.1' + req = swob.Request.blank('/v1/a/c/o', method='POST', + headers={'Content-Type': 'foo/bar', + 'X-Delete-At': t}) + x_newest_responses = [200] * self.obj_ring.replicas + \ + [404] * self.obj_ring.max_more_nodes + with set_http_connect(*x_newest_responses): + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 400) + self.assertEqual('Non-integer X-Delete-At', resp.body) + + def test_POST_delete_at_in_past(self): + t = str(int(time.time() - 100)) + req = swob.Request.blank('/v1/a/c/o', method='POST', + headers={'Content-Type': 'foo/bar', + 'X-Delete-At': t}) + x_newest_responses = [200] * self.obj_ring.replicas + \ + [404] * self.obj_ring.max_more_nodes + with set_http_connect(*x_newest_responses): + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 400) + self.assertEqual('X-Delete-At in past', resp.body) + + def test_PUT_converts_delete_after_to_delete_at(self): + req = swob.Request.blank('/v1/a/c/o', method='PUT', body='', + headers={'Content-Type': 'foo/bar', + 'X-Delete-After': '60'}) + put_headers = [] + + def capture_headers(ip, port, device, part, method, path, headers, + **kwargs): + if method == 'PUT': + put_headers.append(headers) + codes = [201] * self.obj_ring.replicas + t = time.time() + with set_http_connect(*codes, give_connect=capture_headers): + with mock.patch('time.time', lambda: t): + resp = req.get_response(self.app) + self.assertEquals(resp.status_int, 201) + expected_delete_at = str(int(t) + 60) + for given_headers in put_headers: + self.assertEquals(given_headers.get('X-Delete-At'), + expected_delete_at) + self.assertTrue('X-Delete-At-Host' in given_headers) + self.assertTrue('X-Delete-At-Device' in given_headers) + self.assertTrue('X-Delete-At-Partition' in given_headers) + self.assertTrue('X-Delete-At-Container' in given_headers) + + def test_PUT_non_int_delete_after(self): + t = str(int(time.time() + 100)) + '.1' + req = swob.Request.blank('/v1/a/c/o', method='PUT', body='', + headers={'Content-Type': 'foo/bar', + 'X-Delete-After': t}) + with set_http_connect(): + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 400) + self.assertEqual('Non-integer X-Delete-After', resp.body) + + def test_PUT_negative_delete_after(self): + req = swob.Request.blank('/v1/a/c/o', method='PUT', body='', + headers={'Content-Type': 'foo/bar', + 'X-Delete-After': '-60'}) + with set_http_connect(): + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 400) + self.assertEqual('X-Delete-At in past', resp.body) + + def test_PUT_delete_at(self): + t = str(int(time.time() + 100)) + req = swob.Request.blank('/v1/a/c/o', method='PUT', body='', + headers={'Content-Type': 'foo/bar', + 'X-Delete-At': t}) + put_headers = [] + + def capture_headers(ip, port, device, part, method, path, headers, + **kwargs): + if method == 'PUT': + put_headers.append(headers) + codes = [201] * self.obj_ring.replicas + with set_http_connect(*codes, give_connect=capture_headers): + resp = req.get_response(self.app) + self.assertEquals(resp.status_int, 201) + for given_headers in put_headers: + self.assertEquals(given_headers.get('X-Delete-At'), t) + self.assertTrue('X-Delete-At-Host' in given_headers) + self.assertTrue('X-Delete-At-Device' in given_headers) + self.assertTrue('X-Delete-At-Partition' in given_headers) + self.assertTrue('X-Delete-At-Container' in given_headers) + + def test_PUT_delete_at_non_integer(self): + t = str(int(time.time() - 100)) + '.1' + req = swob.Request.blank('/v1/a/c/o', method='PUT', body='', + headers={'Content-Type': 'foo/bar', + 'X-Delete-At': t}) + with set_http_connect(): + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 400) + self.assertEqual('Non-integer X-Delete-At', resp.body) + + def test_PUT_delete_at_in_past(self): + t = str(int(time.time() - 100)) + req = swob.Request.blank('/v1/a/c/o', method='PUT', body='', + headers={'Content-Type': 'foo/bar', + 'X-Delete-At': t}) + with set_http_connect(): + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 400) + self.assertEqual('X-Delete-At in past', resp.body) + def test_container_sync_put_x_timestamp_not_found(self): test_indexes = [None] + [int(p) for p in POLICIES] for policy_index in test_indexes: diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 611926dca2..2124dcc454 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -4125,176 +4125,6 @@ class TestObjectController(unittest.TestCase): finally: time.time = orig_time - def test_POST_non_int_delete_after(self): - with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') - set_http_connect(200, 200, 200, 200, 200, 202, 202, 202) - self.app.memcache.store = {} - req = Request.blank('/v1/a/c/o', {}, - headers={'Content-Type': 'foo/bar', - 'X-Delete-After': '60.1'}) - self.app.update_request(req) - res = controller.POST(req) - self.assertEquals(res.status, '400 Bad Request') - self.assertTrue('Non-integer X-Delete-After' in res.body) - - def test_POST_negative_delete_after(self): - with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') - set_http_connect(200, 200, 200, 200, 200, 202, 202, 202) - self.app.memcache.store = {} - req = Request.blank('/v1/a/c/o', {}, - headers={'Content-Type': 'foo/bar', - 'X-Delete-After': '-60'}) - self.app.update_request(req) - res = controller.POST(req) - self.assertEquals(res.status, '400 Bad Request') - self.assertTrue('X-Delete-At in past' in res.body) - - def test_POST_delete_at(self): - with save_globals(): - given_headers = {} - - def fake_make_requests(req, ring, part, method, path, headers, - query_string=''): - given_headers.update(headers[0]) - - self.app.object_post_as_copy = False - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') - controller.make_requests = fake_make_requests - set_http_connect(200, 200) - self.app.memcache.store = {} - t = str(int(time.time() + 100)) - req = Request.blank('/v1/a/c/o', {}, - headers={'Content-Type': 'foo/bar', - 'X-Delete-At': t}) - self.app.update_request(req) - controller.POST(req) - self.assertEquals(given_headers.get('X-Delete-At'), t) - self.assertTrue('X-Delete-At-Host' in given_headers) - self.assertTrue('X-Delete-At-Device' in given_headers) - self.assertTrue('X-Delete-At-Partition' in given_headers) - self.assertTrue('X-Delete-At-Container' in given_headers) - - t = str(int(time.time() + 100)) + '.1' - req = Request.blank('/v1/a/c/o', {}, - headers={'Content-Type': 'foo/bar', - 'X-Delete-At': t}) - self.app.update_request(req) - resp = controller.POST(req) - self.assertEquals(resp.status_int, 400) - self.assertTrue('Non-integer X-Delete-At' in resp.body) - - t = str(int(time.time() - 100)) - req = Request.blank('/v1/a/c/o', {}, - headers={'Content-Type': 'foo/bar', - 'X-Delete-At': t}) - self.app.update_request(req) - resp = controller.POST(req) - self.assertEquals(resp.status_int, 400) - self.assertTrue('X-Delete-At in past' in resp.body) - - def test_PUT_converts_delete_after_to_delete_at(self): - with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') - set_http_connect(200, 200, 201, 201, 201) - self.app.memcache.store = {} - orig_time = time.time - try: - t = time.time() - time.time = lambda: t - req = Request.blank('/v1/a/c/o', {}, - headers={'Content-Length': '0', - 'Content-Type': 'foo/bar', - 'X-Delete-After': '60'}) - self.app.update_request(req) - res = controller.PUT(req) - self.assertEquals(res.status, '201 Fake') - self.assertEquals(req.headers.get('x-delete-at'), - str(int(t + 60))) - finally: - time.time = orig_time - - def test_PUT_non_int_delete_after(self): - with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') - set_http_connect(200, 200, 201, 201, 201) - self.app.memcache.store = {} - req = Request.blank('/v1/a/c/o', {}, - headers={'Content-Length': '0', - 'Content-Type': 'foo/bar', - 'X-Delete-After': '60.1'}) - self.app.update_request(req) - res = controller.PUT(req) - self.assertEquals(res.status, '400 Bad Request') - self.assertTrue('Non-integer X-Delete-After' in res.body) - - def test_PUT_negative_delete_after(self): - with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') - set_http_connect(200, 200, 201, 201, 201) - self.app.memcache.store = {} - req = Request.blank('/v1/a/c/o', {}, - headers={'Content-Length': '0', - 'Content-Type': 'foo/bar', - 'X-Delete-After': '-60'}) - self.app.update_request(req) - res = controller.PUT(req) - self.assertEquals(res.status, '400 Bad Request') - self.assertTrue('X-Delete-At in past' in res.body) - - def test_PUT_delete_at(self): - with save_globals(): - given_headers = {} - - def fake_connect_put_node(nodes, part, path, headers, - logger_thread_locals): - given_headers.update(headers) - - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') - controller._connect_put_node = fake_connect_put_node - set_http_connect(200, 200) - self.app.memcache.store = {} - t = str(int(time.time() + 100)) - req = Request.blank('/v1/a/c/o', {}, - headers={'Content-Length': '0', - 'Content-Type': 'foo/bar', - 'X-Delete-At': t}) - self.app.update_request(req) - controller.PUT(req) - self.assertEquals(given_headers.get('X-Delete-At'), t) - self.assertTrue('X-Delete-At-Host' in given_headers) - self.assertTrue('X-Delete-At-Device' in given_headers) - self.assertTrue('X-Delete-At-Partition' in given_headers) - self.assertTrue('X-Delete-At-Container' in given_headers) - - t = str(int(time.time() + 100)) + '.1' - req = Request.blank('/v1/a/c/o', {}, - headers={'Content-Length': '0', - 'Content-Type': 'foo/bar', - 'X-Delete-At': t}) - self.app.update_request(req) - resp = controller.PUT(req) - self.assertEquals(resp.status_int, 400) - self.assertTrue('Non-integer X-Delete-At' in resp.body) - - t = str(int(time.time() - 100)) - req = Request.blank('/v1/a/c/o', {}, - headers={'Content-Length': '0', - 'Content-Type': 'foo/bar', - 'X-Delete-At': t}) - self.app.update_request(req) - resp = controller.PUT(req) - self.assertEquals(resp.status_int, 400) - self.assertTrue('X-Delete-At in past' in resp.body) - @patch_policies([ StoragePolicy(0, 'zero', False, object_ring=FakeRing()), StoragePolicy(1, 'one', True, object_ring=FakeRing())