From fef2afd927159af00b6e0c44310b0c7227ed5cc3 Mon Sep 17 00:00:00 2001 From: gholt Date: Mon, 3 Jun 2013 23:50:05 +0000 Subject: [PATCH] Fixed Bug 1187200 See Bug 1187200 for a full description of the problem. Part 1: X-Delete-At-Container added to X-Delete-At-* info This fixes the bug by passing the expiring-objects-account's container name onward to the backend object servers. This is in case the object servers' expiring_objects_container_divisor happens to be different than the proxy server's, we want to make sure the host, partition, and device match up with the container name. Different container names would be fine, but not with mismatched host, partition, and device info. Part 2: The db_replicator now double checks the disk path's partition against the partition the ring gives back. If they don't match, it logs the problem but continues to replicate the database to where it should be and, on success to all proper nodes, removes the local out of place database. Bug 1187200 Change-Id: Id0873a3f2198ce285fe0b0c777738eff38bc2438 --- swift/common/db_replicator.py | 18 ++- swift/obj/server.py | 26 ++++- swift/proxy/controllers/obj.py | 12 +- test/unit/common/test_db_replicator.py | 52 ++++++++- test/unit/obj/test_server.py | 146 ++++++++++++++++++++----- test/unit/proxy/test_server.py | 30 ++++- 6 files changed, 241 insertions(+), 43 deletions(-) diff --git a/swift/common/db_replicator.py b/swift/common/db_replicator.py index 9391c7cd71..96c64f78ab 100644 --- a/swift/common/db_replicator.py +++ b/swift/common/db_replicator.py @@ -28,6 +28,7 @@ from eventlet.green import subprocess import simplejson import swift.common.db +from swift.common.direct_client import quote from swift.common.utils import get_logger, whataremyips, storage_directory, \ renamer, mkdirs, lock_parent_directory, config_true_value, \ unlink_older_than, dump_recon_cache, rsync_ip @@ -408,12 +409,26 @@ class Replicator(Daemon): self.logger.debug(_('Replicating db %s'), object_file) self.stats['attempted'] += 1 self.logger.increment('attempts') + shouldbehere = True try: broker = self.brokerclass(object_file, pending_timeout=30) broker.reclaim(time.time() - self.reclaim_age, time.time() - (self.reclaim_age * 2)) info = broker.get_replication_info() full_info = broker.get_info() + bpart = self.ring.get_part( + full_info['account'], full_info.get('container')) + if bpart != int(partition): + partition = bpart + # Important to set this false here since the later check only + # checks if it's on the proper device, not partition. + shouldbehere = False + name = '/' + quote(full_info['account']) + if 'container' in full_info: + name += '/' + quote(full_info['container']) + self.logger.error( + 'Found %s for %s when it should be on partition %s; will ' + 'replicate out and remove.' % (object_file, name, bpart)) except (Exception, Timeout), e: if 'no such table' in str(e): self.logger.error(_('Quarantining DB %s'), object_file) @@ -444,7 +459,8 @@ class Replicator(Daemon): return responses = [] nodes = self.ring.get_part_nodes(int(partition)) - shouldbehere = bool([n for n in nodes if n['id'] == node_id]) + if shouldbehere: + shouldbehere = bool([n for n in nodes if n['id'] == node_id]) # See Footnote [1] for an explanation of the repl_nodes assignment. i = 0 while i < len(nodes) and nodes[i]['id'] != node_id: diff --git a/swift/obj/server.py b/swift/obj/server.py index a210f02543..5fd7b94e0e 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -623,6 +623,17 @@ class ObjectController(object): 'x-trans-id': headers_in.get('x-trans-id', '-'), 'referer': request.as_referer()}) if op != 'DELETE': + delete_at_container = headers_in.get('X-Delete-At-Container', None) + if not delete_at_container: + self.logger.warning( + 'X-Delete-At-Container header must be specified for ' + 'expiring objects background %s to work properly. Making ' + 'best guess as to the container name for now.' % op) + # TODO(gholt): In a future release, change the above warning to + # a raised exception and remove the guess code below. + delete_at_container = str( + delete_at / self.expiring_objects_container_divisor * + self.expiring_objects_container_divisor) partition = headers_in.get('X-Delete-At-Partition', None) hosts = headers_in.get('X-Delete-At-Host', '') contdevices = headers_in.get('X-Delete-At-Device', '') @@ -635,12 +646,21 @@ class ObjectController(object): headers_out['x-size'] = '0' headers_out['x-content-type'] = 'text/plain' headers_out['x-etag'] = 'd41d8cd98f00b204e9800998ecf8427e' + else: + # DELETEs of old expiration data have no way of knowing what the + # old X-Delete-At-Container was at the time of the initial setting + # of the data, so a best guess is made here. + # Worst case is a DELETE is issued now for something that doesn't + # exist there and the original data is left where it is, where + # it will be ignored when the expirer eventually tries to issue the + # object DELETE later since the X-Delete-At value won't match up. + delete_at_container = str( + delete_at / self.expiring_objects_container_divisor * + self.expiring_objects_container_divisor) for host, contdevice in updates: self.async_update( - op, self.expiring_objects_account, - str(delete_at / self.expiring_objects_container_divisor * - self.expiring_objects_container_divisor), + op, self.expiring_objects_account, delete_at_container, '%s-%s/%s/%s' % (delete_at, account, container, obj), host, partition, contdevice, headers_out, objdevice) diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index 633d9abc41..5174996e88 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -597,14 +597,14 @@ class ObjectController(Controller): self.app.container_ring.get_nodes( self.app.expiring_objects_account, delete_at_container) else: - delete_at_part = delete_at_nodes = None + delete_at_container = delete_at_part = delete_at_nodes = None partition, nodes = self.app.object_ring.get_nodes( self.account_name, self.container_name, self.object_name) req.headers['X-Timestamp'] = normalize_timestamp(time.time()) headers = self._backend_requests( req, len(nodes), container_partition, containers, - delete_at_part, delete_at_nodes) + delete_at_container, delete_at_part, delete_at_nodes) resp = self.make_requests(req, self.app.object_ring, partition, 'POST', req.path_info, headers) @@ -612,7 +612,8 @@ class ObjectController(Controller): def _backend_requests(self, req, n_outgoing, container_partition, containers, - delete_at_partition=None, delete_at_nodes=None): + delete_at_container=None, delete_at_partition=None, + delete_at_nodes=None): headers = [self.generate_request_headers(req, additional=req.headers) for _junk in range(n_outgoing)] @@ -633,6 +634,7 @@ class ObjectController(Controller): for i, node in enumerate(delete_at_nodes or []): i = i % len(headers) + headers[i]['X-Delete-At-Container'] = delete_at_container headers[i]['X-Delete-At-Partition'] = delete_at_partition headers[i]['X-Delete-At-Host'] = csv_append( headers[i].get('X-Delete-At-Host'), @@ -872,7 +874,7 @@ class ObjectController(Controller): self.app.container_ring.get_nodes( self.app.expiring_objects_account, delete_at_container) else: - delete_at_part = delete_at_nodes = None + delete_at_container = delete_at_part = delete_at_nodes = None node_iter = GreenthreadSafeIterator( self.iter_nodes(self.app.object_ring, partition)) @@ -882,7 +884,7 @@ class ObjectController(Controller): outgoing_headers = self._backend_requests( req, len(nodes), container_partition, containers, - delete_at_part, delete_at_nodes) + delete_at_container, delete_at_part, delete_at_nodes) for nheaders in outgoing_headers: # RFC2616:8.2.3 disallows 100-continue without a body diff --git a/test/unit/common/test_db_replicator.py b/test/unit/common/test_db_replicator.py index 43e9b4a02f..5c4a114246 100644 --- a/test/unit/common/test_db_replicator.py +++ b/test/unit/common/test_db_replicator.py @@ -30,6 +30,10 @@ from swift.container import server as container_server from test.unit import FakeLogger +TEST_ACCOUNT_NAME = 'a c t' +TEST_CONTAINER_NAME = 'c o n' + + def teardown_module(): "clean up my monkey patching" reload(db_replicator) @@ -47,6 +51,9 @@ class FakeRing: def __init__(self, path, reload_time=15, ring_name=None): pass + def get_part(self, account, container=None, obj=None): + return 0 + def get_part_nodes(self, part): return [] @@ -72,6 +79,9 @@ class FakeRingWithNodes: def __init__(self, path, reload_time=15, ring_name=None): pass + def get_part(self, account, container=None, obj=None): + return 0 + def get_part_nodes(self, part): return self.devs[:3] @@ -139,6 +149,7 @@ class FakeBroker: get_repl_missing_table = False stub_replication_info = None db_type = 'container' + info = {'account': TEST_ACCOUNT_NAME, 'container': TEST_CONTAINER_NAME} def __init__(self, *args, **kwargs): self.locked = False @@ -178,7 +189,12 @@ class FakeBroker: pass def get_info(self): - pass + return self.info + + +class FakeAccountBroker(FakeBroker): + db_type = 'account' + info = {'account': TEST_ACCOUNT_NAME} class TestReplicator(db_replicator.Replicator): @@ -453,6 +469,40 @@ class TestDBReplicator(unittest.TestCase): replicator._replicate_object('0', '/path/to/file', 'node_id') self.assertEquals(['/path/to/file'], self.delete_db_calls) + def test_replicate_account_out_of_place(self): + replicator = TestReplicator({}) + replicator.ring = FakeRingWithNodes().Ring('path') + replicator.brokerclass = FakeAccountBroker + replicator._repl_to_node = lambda *args: True + replicator.delete_db = self.stub_delete_db + replicator.logger = FakeLogger() + # Correct node_id, wrong part + part = replicator.ring.get_part(TEST_ACCOUNT_NAME) + 1 + node_id = replicator.ring.get_part_nodes(part)[0]['id'] + replicator._replicate_object(str(part), '/path/to/file', node_id) + self.assertEqual(['/path/to/file'], self.delete_db_calls) + self.assertEqual( + replicator.logger.log_dict['error'], + [(('Found /path/to/file for /a%20c%20t when it should be on ' + 'partition 0; will replicate out and remove.',), {})]) + + def test_replicate_container_out_of_place(self): + replicator = TestReplicator({}) + replicator.ring = FakeRingWithNodes().Ring('path') + replicator._repl_to_node = lambda *args: True + replicator.delete_db = self.stub_delete_db + replicator.logger = FakeLogger() + # Correct node_id, wrong part + part = replicator.ring.get_part( + TEST_ACCOUNT_NAME, TEST_CONTAINER_NAME) + 1 + node_id = replicator.ring.get_part_nodes(part)[0]['id'] + replicator._replicate_object(str(part), '/path/to/file', node_id) + self.assertEqual(['/path/to/file'], self.delete_db_calls) + self.assertEqual( + replicator.logger.log_dict['error'], + [(('Found /path/to/file for /a%20c%20t/c%20o%20n when it should ' + 'be on partition 0; will replicate out and remove.',), {})]) + def test_delete_db(self): db_replicator.lock_parent_directory = lock_parent_directory replicator = TestReplicator({}) diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index 647d955d7f..cb98f3efb8 100755 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -1771,6 +1771,7 @@ class TestObjectController(unittest.TestCase): 'X-Container-Host': '1.2.3.4:5', 'X-Container-Device': 'sdb1', 'X-Delete-At': 9999999999, + 'X-Delete-At-Container': '9999999960', 'X-Delete-At-Host': "10.1.1.1:6001,10.2.2.2:6002", 'X-Delete-At-Partition': '6237', 'X-Delete-At-Device': 'sdp,sdq'}) @@ -2047,7 +2048,9 @@ class TestObjectController(unittest.TestCase): 'x-trans-id': '123', 'referer': 'PUT http://localhost/v1/a/c/o'}, 'sda1']) - def test_delete_at_update_put(self): + def test_delete_at_update_on_put(self): + # Test how delete_at_update works when issued a delete for old + # expiration info after a new put with no new expiration info. given_args = [] def fake_async_update(*args): @@ -2058,17 +2061,17 @@ class TestObjectController(unittest.TestCase): environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': 1, 'X-Trans-Id': '123'}) - self.object_controller.delete_at_update('PUT', 2, 'a', 'c', 'o', + self.object_controller.delete_at_update('DELETE', 2, 'a', 'c', 'o', req, 'sda1') - self.assertEquals(given_args, ['PUT', '.expiring_objects', '0', + self.assertEquals(given_args, ['DELETE', '.expiring_objects', '0', '2-a/c/o', None, None, None, - HeaderKeyDict({'x-size': '0', - 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', - 'x-content-type': 'text/plain', 'x-timestamp': '1', + HeaderKeyDict({'x-timestamp': '1', 'x-trans-id': '123', 'referer': 'PUT http://localhost/v1/a/c/o'}), 'sda1']) def test_delete_at_negative(self): + # Test how delete_at_update works when issued a delete for old + # expiration info after a new put with no new expiration info. # Test negative is reset to 0 given_args = [] @@ -2081,16 +2084,16 @@ class TestObjectController(unittest.TestCase): headers={'X-Timestamp': 1, 'X-Trans-Id': '1234'}) self.object_controller.delete_at_update( - 'PUT', -2, 'a', 'c', 'o', req, 'sda1') + 'DELETE', -2, 'a', 'c', 'o', req, 'sda1') self.assertEquals(given_args, [ - 'PUT', '.expiring_objects', '0', '0-a/c/o', None, None, None, - HeaderKeyDict({'x-size': '0', - 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', - 'x-content-type': 'text/plain', 'x-timestamp': '1', + 'DELETE', '.expiring_objects', '0', '0-a/c/o', None, None, None, + HeaderKeyDict({'x-timestamp': '1', 'x-trans-id': '1234', 'referer': 'PUT http://localhost/v1/a/c/o'}), 'sda1']) def test_delete_at_cap(self): + # Test how delete_at_update works when issued a delete for old + # expiration info after a new put with no new expiration info. # Test past cap is reset to cap given_args = [] @@ -2103,17 +2106,18 @@ class TestObjectController(unittest.TestCase): headers={'X-Timestamp': 1, 'X-Trans-Id': '1234'}) self.object_controller.delete_at_update( - 'PUT', 12345678901, 'a', 'c', 'o', req, 'sda1') + 'DELETE', 12345678901, 'a', 'c', 'o', req, 'sda1') self.assertEquals(given_args, [ - 'PUT', '.expiring_objects', '9999936000', '9999999999-a/c/o', None, - None, None, - HeaderKeyDict({'x-size': '0', - 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', - 'x-content-type': 'text/plain', 'x-timestamp': '1', + 'DELETE', '.expiring_objects', '9999936000', '9999999999-a/c/o', + None, None, None, + HeaderKeyDict({'x-timestamp': '1', 'x-trans-id': '1234', 'referer': 'PUT http://localhost/v1/a/c/o'}), 'sda1']) def test_delete_at_update_put_with_info(self): + # Keep next test, + # test_delete_at_update_put_with_info_but_missing_container, in sync + # with this one but just missing the X-Delete-At-Container header. given_args = [] def fake_async_update(*args): @@ -2124,6 +2128,7 @@ class TestObjectController(unittest.TestCase): environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': 1, 'X-Trans-Id': '1234', + 'X-Delete-At-Container': '0', 'X-Delete-At-Host': '127.0.0.1:1234', 'X-Delete-At-Partition': '3', 'X-Delete-At-Device': 'sdc1'}) @@ -2137,6 +2142,31 @@ class TestObjectController(unittest.TestCase): 'x-trans-id': '1234', 'referer': 'PUT http://localhost/v1/a/c/o'}), 'sda1']) + def test_delete_at_update_put_with_info_but_missing_container(self): + # Same as previous test, test_delete_at_update_put_with_info, but just + # missing the X-Delete-At-Container header. + given_args = [] + + def fake_async_update(*args): + given_args.extend(args) + + self.object_controller.async_update = fake_async_update + self.object_controller.logger = FakeLogger() + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': 1, + 'X-Trans-Id': '1234', + 'X-Delete-At-Host': '127.0.0.1:1234', + 'X-Delete-At-Partition': '3', + 'X-Delete-At-Device': 'sdc1'}) + self.object_controller.delete_at_update('PUT', 2, 'a', 'c', 'o', + req, 'sda1') + self.assertEquals( + self.object_controller.logger.log_dict['warning'], + [(('X-Delete-At-Container header must be specified for expiring ' + 'objects background PUT to work properly. Making best guess as ' + 'to the container name for now.',), {})]) + def test_delete_at_update_delete(self): given_args = [] @@ -2269,9 +2299,15 @@ class TestObjectController(unittest.TestCase): def test_GET_but_expired(self): test_time = time() + 10000 + delete_at_timestamp = int(test_time + 100) + delete_at_container = str( + delete_at_timestamp / + self.object_controller.expiring_objects_container_divisor * + self.object_controller.expiring_objects_container_divisor) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(test_time - 2000), - 'X-Delete-At': str(int(test_time + 100)), + 'X-Delete-At': str(delete_at_timestamp), + 'X-Delete-At-Container': delete_at_container, 'Content-Length': '4', 'Content-Type': 'application/octet-stream'}) req.body = 'TEST' @@ -2287,10 +2323,16 @@ class TestObjectController(unittest.TestCase): try: t = time() object_server.time.time = lambda: t + delete_at_timestamp = int(t + 1) + delete_at_container = str( + delete_at_timestamp / + self.object_controller.expiring_objects_container_divisor * + self.object_controller.expiring_objects_container_divisor) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(test_time - 1000), - 'X-Delete-At': str(int(t + 1)), + 'X-Delete-At': str(delete_at_timestamp), + 'X-Delete-At-Container': delete_at_container, 'Content-Length': '4', 'Content-Type': 'application/octet-stream'}) req.body = 'TEST' @@ -2318,9 +2360,15 @@ class TestObjectController(unittest.TestCase): def test_HEAD_but_expired(self): test_time = time() + 10000 + delete_at_timestamp = int(test_time + 100) + delete_at_container = str( + delete_at_timestamp / + self.object_controller.expiring_objects_container_divisor * + self.object_controller.expiring_objects_container_divisor) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(test_time - 2000), - 'X-Delete-At': str(int(test_time + 100)), + 'X-Delete-At': str(delete_at_timestamp), + 'X-Delete-At-Container': delete_at_container, 'Content-Length': '4', 'Content-Type': 'application/octet-stream'}) req.body = 'TEST' @@ -2336,11 +2384,17 @@ class TestObjectController(unittest.TestCase): orig_time = object_server.time.time try: t = time() + delete_at_timestamp = int(t + 1) + delete_at_container = str( + delete_at_timestamp / + self.object_controller.expiring_objects_container_divisor * + self.object_controller.expiring_objects_container_divisor) object_server.time.time = lambda: t req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(test_time - 1000), - 'X-Delete-At': str(int(t + 1)), + 'X-Delete-At': str(delete_at_timestamp), + 'X-Delete-At-Container': delete_at_container, 'Content-Length': '4', 'Content-Type': 'application/octet-stream'}) req.body = 'TEST' @@ -2368,9 +2422,15 @@ class TestObjectController(unittest.TestCase): def test_POST_but_expired(self): test_time = time() + 10000 + delete_at_timestamp = int(test_time + 100) + delete_at_container = str( + delete_at_timestamp / + self.object_controller.expiring_objects_container_divisor * + self.object_controller.expiring_objects_container_divisor) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(test_time - 2000), - 'X-Delete-At': str(int(test_time + 100)), + 'X-Delete-At': str(delete_at_timestamp), + 'X-Delete-At-Container': delete_at_container, 'Content-Length': '4', 'Content-Type': 'application/octet-stream'}) req.body = 'TEST' @@ -2383,9 +2443,15 @@ class TestObjectController(unittest.TestCase): resp = self.object_controller.POST(req) self.assertEquals(resp.status_int, 202) + delete_at_timestamp = int(time() + 1) + delete_at_container = str( + delete_at_timestamp / + self.object_controller.expiring_objects_container_divisor * + self.object_controller.expiring_objects_container_divisor) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(test_time - 1000), - 'X-Delete-At': str(int(time() + 1)), + 'X-Delete-At': str(delete_at_timestamp), + 'X-Delete-At-Container': delete_at_container, 'Content-Length': '4', 'Content-Type': 'application/octet-stream'}) req.body = 'TEST' @@ -2406,9 +2472,15 @@ class TestObjectController(unittest.TestCase): def test_DELETE_but_expired(self): test_time = time() + 10000 + delete_at_timestamp = int(test_time + 100) + delete_at_container = str( + delete_at_timestamp / + self.object_controller.expiring_objects_container_divisor * + self.object_controller.expiring_objects_container_divisor) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(test_time - 2000), - 'X-Delete-At': str(int(test_time + 100)), + 'X-Delete-At': str(delete_at_timestamp), + 'X-Delete-At-Container': delete_at_container, 'Content-Length': '4', 'Content-Type': 'application/octet-stream'}) req.body = 'TEST' @@ -2443,9 +2515,15 @@ class TestObjectController(unittest.TestCase): resp = self.object_controller.DELETE(req) self.assertEquals(resp.status_int, 204) + delete_at_timestamp = int(test_time - 1) + delete_at_container = str( + delete_at_timestamp / + self.object_controller.expiring_objects_container_divisor * + self.object_controller.expiring_objects_container_divisor) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(test_time - 97), - 'X-Delete-At': str(int(test_time - 1)), + 'X-Delete-At': str(delete_at_timestamp), + 'X-Delete-At-Container': delete_at_container, 'Content-Length': '4', 'Content-Type': 'application/octet-stream'}) req.body = 'TEST' @@ -2465,10 +2543,15 @@ class TestObjectController(unittest.TestCase): resp = self.object_controller.DELETE(req) self.assertEquals(resp.status_int, 204) - delete_at_timestamp = str(int(test_time - 1)) + delete_at_timestamp = int(test_time - 1) + delete_at_container = str( + delete_at_timestamp / + self.object_controller.expiring_objects_container_divisor * + self.object_controller.expiring_objects_container_divisor) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(test_time - 94), - 'X-Delete-At': delete_at_timestamp, + 'X-Delete-At': str(delete_at_timestamp), + 'X-Delete-At-Container': delete_at_container, 'Content-Length': '4', 'Content-Type': 'application/octet-stream'}) req.body = 'TEST' @@ -2498,12 +2581,17 @@ class TestObjectController(unittest.TestCase): self.object_controller.delete_at_update = fake_delete_at_update timestamp1 = normalize_timestamp(time()) - delete_at_timestamp1 = str(int(time() + 1000)) + delete_at_timestamp1 = int(time() + 1000) + delete_at_container1 = str( + delete_at_timestamp1 / + self.object_controller.expiring_objects_container_divisor * + self.object_controller.expiring_objects_container_divisor) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp1, 'Content-Length': '4', 'Content-Type': 'application/octet-stream', - 'X-Delete-At': delete_at_timestamp1}) + 'X-Delete-At': str(delete_at_timestamp1), + 'X-Delete-At-Container': delete_at_container1}) req.body = 'TEST' resp = self.object_controller.PUT(req) self.assertEquals(resp.status_int, 201) diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 56233d2b22..251d0e1d23 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -58,6 +58,7 @@ from swift.common.swob import Request, Response, HTTPNotFound, \ logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) +STATIC_TIME = time.time() _request_instances = 0 @@ -2844,6 +2845,7 @@ class TestObjectController(unittest.TestCase): self.assertTrue('X-Delete-At-Host' in given_headers) self.assertTrue('X-Delete-At-Device' in given_headers) self.assertTrue('X-Delete-At-Partition' in given_headers) + self.assertTrue('X-Delete-At-Container' in given_headers) def test_chunked_put(self): @@ -4102,6 +4104,7 @@ class TestObjectController(unittest.TestCase): self.assertTrue('X-Delete-At-Host' in given_headers) self.assertTrue('X-Delete-At-Device' in given_headers) self.assertTrue('X-Delete-At-Partition' in given_headers) + self.assertTrue('X-Delete-At-Container' in given_headers) t = str(int(time.time() + 100)) + '.1' req = Request.blank('/a/c/o', {}, @@ -4197,6 +4200,7 @@ class TestObjectController(unittest.TestCase): self.assertTrue('X-Delete-At-Host' in given_headers) self.assertTrue('X-Delete-At-Device' in given_headers) self.assertTrue('X-Delete-At-Partition' in given_headers) + self.assertTrue('X-Delete-At-Container' in given_headers) t = str(int(time.time() + 100)) + '.1' req = Request.blank('/a/c/o', {}, @@ -4537,54 +4541,72 @@ class TestObjectController(unittest.TestCase): 'X-Container-Partition': '1', 'X-Container-Device': 'sdc'}]) + @mock.patch('time.time', new=lambda: STATIC_TIME) def test_PUT_x_delete_at_with_fewer_container_replicas(self): self.app.container_ring.set_replicas(2) + delete_at_timestamp = int(time.time()) + 100000 + delete_at_container = str( + delete_at_timestamp / + self.app.expiring_objects_container_divisor * + self.app.expiring_objects_container_divisor) req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Type': 'application/stuff', 'Content-Length': '0', - 'X-Delete-At': int(time.time()) + 100000}) + 'X-Delete-At': str(delete_at_timestamp)}) controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') seen_headers = self._gather_x_container_headers( controller.PUT, req, 200, 200, 201, 201, 201, # HEAD HEAD PUT PUT PUT header_list=('X-Delete-At-Host', 'X-Delete-At-Device', - 'X-Delete-At-Partition')) + 'X-Delete-At-Partition', 'X-Delete-At-Container')) self.assertEqual(seen_headers, [ {'X-Delete-At-Host': '10.0.0.0:1000', + 'X-Delete-At-Container': delete_at_container, 'X-Delete-At-Partition': '1', 'X-Delete-At-Device': 'sda'}, {'X-Delete-At-Host': '10.0.0.1:1001', + 'X-Delete-At-Container': delete_at_container, 'X-Delete-At-Partition': '1', 'X-Delete-At-Device': 'sdb'}, {'X-Delete-At-Host': None, + 'X-Delete-At-Container': None, 'X-Delete-At-Partition': None, 'X-Delete-At-Device': None}]) + @mock.patch('time.time', new=lambda: STATIC_TIME) def test_PUT_x_delete_at_with_more_container_replicas(self): self.app.container_ring.set_replicas(4) self.app.expiring_objects_account = 'expires' self.app.expiring_objects_container_divisor = 60 + delete_at_timestamp = int(time.time()) + 100000 + delete_at_container = str( + delete_at_timestamp / + self.app.expiring_objects_container_divisor * + self.app.expiring_objects_container_divisor) req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Type': 'application/stuff', 'Content-Length': 0, - 'X-Delete-At': int(time.time()) + 100000}) + 'X-Delete-At': str(delete_at_timestamp)}) controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') seen_headers = self._gather_x_container_headers( controller.PUT, req, 200, 200, 201, 201, 201, # HEAD HEAD PUT PUT PUT header_list=('X-Delete-At-Host', 'X-Delete-At-Device', - 'X-Delete-At-Partition')) + 'X-Delete-At-Partition', 'X-Delete-At-Container')) self.assertEqual(seen_headers, [ {'X-Delete-At-Host': '10.0.0.0:1000,10.0.0.3:1003', + 'X-Delete-At-Container': delete_at_container, 'X-Delete-At-Partition': '1', 'X-Delete-At-Device': 'sda,sdd'}, {'X-Delete-At-Host': '10.0.0.1:1001', + 'X-Delete-At-Container': delete_at_container, 'X-Delete-At-Partition': '1', 'X-Delete-At-Device': 'sdb'}, {'X-Delete-At-Host': '10.0.0.2:1002', + 'X-Delete-At-Container': delete_at_container, 'X-Delete-At-Partition': '1', 'X-Delete-At-Device': 'sdc'}])