diff --git a/swift/common/utils.py b/swift/common/utils.py index 37fbeaea00..f7d70f1c04 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -2960,3 +2960,13 @@ def quote(value, safe='/'): Patched version of urllib.quote that encodes utf-8 strings before quoting """ return _quote(get_valid_utf8_str(value), safe) + + +def get_expirer_container(x_delete_at, expirer_divisor, acc, cont, obj): + """ + Returns a expiring object container name for given X-Delete-At and + a/c/o. + """ + shard_int = int(hash_path(acc, cont, obj), 16) % 100 + return normalize_delete_at_timestamp( + int(x_delete_at) / expirer_divisor * expirer_divisor - shard_int) diff --git a/swift/obj/server.py b/swift/obj/server.py index dc29758563..0fa1d7622e 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -29,7 +29,8 @@ from eventlet import sleep, Timeout from swift.common.utils import public, get_logger, \ config_true_value, timing_stats, replication, \ - normalize_delete_at_timestamp, get_log_line, Timestamp + normalize_delete_at_timestamp, get_log_line, Timestamp, \ + get_expirer_container from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_object_creation, \ valid_timestamp, check_utf8 @@ -284,9 +285,9 @@ class ObjectController(object): 'best guess as to the container name for now.' % op) # TODO(gholt): In a future release, change the above warning to # a raised exception and remove the guess code below. - delete_at_container = ( - int(delete_at) / self.expiring_objects_container_divisor * - self.expiring_objects_container_divisor) + delete_at_container = get_expirer_container( + delete_at, self.expiring_objects_container_divisor, + account, container, obj) partition = headers_in.get('X-Delete-At-Partition', None) hosts = headers_in.get('X-Delete-At-Host', '') contdevices = headers_in.get('X-Delete-At-Device', '') @@ -307,9 +308,9 @@ class ObjectController(object): # exist there and the original data is left where it is, where # it will be ignored when the expirer eventually tries to issue the # object DELETE later since the X-Delete-At value won't match up. - delete_at_container = str( - int(delete_at) / self.expiring_objects_container_divisor * - self.expiring_objects_container_divisor) + delete_at_container = get_expirer_container( + delete_at, self.expiring_objects_container_divisor, + account, container, obj) delete_at_container = normalize_delete_at_timestamp( delete_at_container) diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index 7b3eba440e..b10e2dd18f 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -38,7 +38,7 @@ from eventlet.timeout import Timeout from swift.common.utils import ( clean_content_type, config_true_value, ContextPool, csv_append, GreenAsyncPile, GreenthreadSafeIterator, json, Timestamp, - normalize_delete_at_timestamp, public, quorum_size) + normalize_delete_at_timestamp, public, quorum_size, get_expirer_container) from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_metadata, check_object_creation, \ check_copy_from_header @@ -285,6 +285,7 @@ class ObjectController(Controller): req.headers['X-Backend-Storage-Policy-Index'] = policy_index partition, nodes = obj_ring.get_nodes( self.account_name, self.container_name, self.object_name) + req.headers['X-Timestamp'] = Timestamp(time.time()).internal headers = self._backend_requests( @@ -449,10 +450,11 @@ class ObjectController(Controller): req.environ.setdefault('swift.log_info', []).append( 'x-delete-at:%s' % x_delete_at) - delete_at_container = normalize_delete_at_timestamp( - x_delete_at / - self.app.expiring_objects_container_divisor * - self.app.expiring_objects_container_divisor) + + delete_at_container = get_expirer_container( + x_delete_at, self.app.expiring_objects_container_divisor, + self.account_name, self.container_name, self.object_name) + delete_at_part, delete_at_nodes = \ self.app.container_ring.get_nodes( self.app.expiring_objects_account, delete_at_container) diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index 4637e88fa3..e2137484ef 100755 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -3144,8 +3144,14 @@ class TestObjectController(unittest.TestCase): 'X-Trans-Id': '1234'}) self.object_controller.delete_at_update( 'DELETE', 12345678901, 'a', 'c', 'o', req, 'sda1', 0) + expiring_obj_container = given_args.pop(2) + expected_exp_cont = utils.get_expirer_container( + utils.normalize_delete_at_timestamp(12345678901), + 86400, 'a', 'c', 'o') + self.assertEqual(expiring_obj_container, expected_exp_cont) + self.assertEquals(given_args, [ - 'DELETE', '.expiring_objects', '9999936000', '9999999999-a/c/o', + 'DELETE', '.expiring_objects', '9999999999-a/c/o', None, None, None, HeaderKeyDict({ 'X-Backend-Storage-Policy-Index': 0, diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 65a476ff14..7a09516c8a 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -4810,10 +4810,9 @@ class TestObjectController(unittest.TestCase): self.app.container_ring.set_replicas(2) delete_at_timestamp = int(time.time()) + 100000 - delete_at_container = str( - delete_at_timestamp / - self.app.expiring_objects_container_divisor * - self.app.expiring_objects_container_divisor) + delete_at_container = utils.get_expirer_container( + delete_at_timestamp, self.app.expiring_objects_container_divisor, + 'a', 'c', 'o') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Type': 'application/stuff', 'Content-Length': '0', @@ -4847,10 +4846,9 @@ class TestObjectController(unittest.TestCase): self.app.expiring_objects_container_divisor = 60 delete_at_timestamp = int(time.time()) + 100000 - delete_at_container = str( - delete_at_timestamp / - self.app.expiring_objects_container_divisor * - self.app.expiring_objects_container_divisor) + delete_at_container = utils.get_expirer_container( + delete_at_timestamp, self.app.expiring_objects_container_divisor, + 'a', 'c', 'o') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Type': 'application/stuff', 'Content-Length': 0,