From 5961ba0ca7ddcfe296627fbef01ec4b32d538290 Mon Sep 17 00:00:00 2001 From: Mandell Degerness Date: Wed, 22 Feb 2023 12:14:05 -0800 Subject: [PATCH] expirer: account and container level delay_reaping The object expirer can be configured to delay the reaping of objects from disk after their expiration time using account and container level delay_reaping values. The delay_reaping value of accounts and containers in seconds is configured in the object server config. The object expirer references these configured values to only reap objects from specified accounts and containers after their corresponding delays. The goal of the delay_reaping feature is to prevent accidental or premature data loss if an object marked for deletion with the 'x-delete-at' feature should not be reaped immediately, for whatever reason. Configuring the delay_reaping value at a granular account and container level is beneficial for being able to keep storage capacity consumption in control while maintaining a desired data recovery window. This patch also adds a sample configuration, documentation, and tests for bad configurations and grace period functionality. Co-Authored-By: Anish Kachinthaya Change-Id: I106103438c4162a561486ac73a09436e998ae1f0 --- doc/source/config/object_server_config.rst | 15 + doc/source/overview_expiring_objects.rst | 43 ++ etc/object-expirer.conf-sample | 21 + etc/object-server.conf-sample | 25 +- swift/obj/expirer.py | 63 ++- test/unit/obj/test_expirer.py | 443 +++++++++++++++++++++ 6 files changed, 605 insertions(+), 5 deletions(-) diff --git a/doc/source/config/object_server_config.rst b/doc/source/config/object_server_config.rst index 414be1dafb..b6b6b0991a 100644 --- a/doc/source/config/object_server_config.rst +++ b/doc/source/config/object_server_config.rst @@ -746,4 +746,19 @@ ionice_priority None I/O scheduling pri priority of the process. Work only with ionice_class. Ignored if IOPRIO_CLASS_IDLE is set. +delay_reaping_ 0.0 A dynamic configuration option for + setting account level delay_reaping values. + The delay_reaping value is configured for + the account with the name placed in + . The object expirer will reap objects in + this account from disk only after this delay + following their x-delete-at time. +delay_reaping_/ 0.0 A dynamic configuration option for + setting container level delay_reaping values. + The delay_reaping value is configured for + the container with the account name placed + in and the container name in . + The object expirer will reap objects in this + container from disk only after this delay + following their x-delete-at time. ============================= =============================== ========================================== diff --git a/doc/source/overview_expiring_objects.rst b/doc/source/overview_expiring_objects.rst index 78d8d3e3b7..ef39d7ba74 100644 --- a/doc/source/overview_expiring_objects.rst +++ b/doc/source/overview_expiring_objects.rst @@ -55,6 +55,49 @@ it will then look for and use the ``/etc/swift/object-expirer.conf`` config. The latter config file is considered deprecated and is searched for to aid in cluster upgrades. +Delay Reaping of Objects from Disk +---------------------------------- + +Swift's expiring object ``x-delete-at`` feature can be used to have the cluster +reap user's objects automatically from disk on their behalf when they no longer +want them stored in their account. In some cases it may be necessary to +"intervene" in the expected expiration process to prevent accidental or +premature data loss if an object marked for expiration should NOT be deleted +immediately when it expires for whatever reason. In these cases +``swift-object-expirer`` offers configuration of a ``delay_reaping`` value +on accounts and containers, which provides a delay between when an object +is marked for deletion, or expired, and when it is actually reaped from disk. +When this is set in the object expirer config the object expirer leaves expired +objects on disk (and in container listings) for the ``delay_reaping`` time. +After this delay has passed objects will be reaped as normal. + +The ``delay_reaping`` value can be set either at an account level or a +container level. When set at an account level, the object expirer will +only reap objects within the account after the delay. A container level +``delay_reaping`` works similarly for containers and overrides an account +level ``delay_reaping`` value. + +The ``delay_reaping`` values are set in the ``[object-expirer]`` section in +either the object-server or object-expirer config files. They are configured +with dynamic config option names prefixed with ``delay_reaping_`` +at the account level and ``delay_reaping_/`` at the container +level, with the ``delay_reaping`` value in seconds. + +Here is an example of ``delay_reaping`` configs in the``object-expirer`` +section in the ``object-server.conf``:: + + [object-expirer] + delay_reaping_AUTH_test = 300.0 + delay_reaping_AUTH_test2 = 86400.0 + delay_reaping_AUTH_test/test = 0.0 + delay_reaping_AUTH_test/test2 = 600.0 + +.. note:: + A container level ``delay_reaping`` value does not require an account level + ``delay_reaping`` value but overrides the account level value for the same + account if it exists. By default, no ``delay_reaping`` value is configured + for any accounts or containers. + Upgrading impact: General Task Queue vs Legacy Queue ---------------------------------------------------- diff --git a/etc/object-expirer.conf-sample b/etc/object-expirer.conf-sample index a83d9e0013..d722dcf9da 100644 --- a/etc/object-expirer.conf-sample +++ b/etc/object-expirer.conf-sample @@ -72,6 +72,27 @@ # queue. # reclaim_age = 604800 # +# The expirer can delay the reaping of expired objects on disk (and in +# container listings) with an account level or container level delay_reaping +# time. +# After the delay_reaping time has passed objects will be reaped as normal. +# You may configure this delay_reaping value in seconds with dynamic config +# option names prefixed with delay_reaping_ for account level delays +# and delay_reaping_/ for container level delays. +# Special characters in or should be quoted. +# The delay_reaping value should be a float value greater than or equal to +# zero. +# A container level delay_reaping does not require an account level +# delay_reaping but overrides the account level delay_reaping for the same +# account if it exists. +# For example: +# delay_reaping_AUTH_test = 300.0 +# delay_reaping_AUTH_test2 = 86400.0 +# delay_reaping_AUTH_test/test = 400.0 +# delay_reaping_AUTH_test/test2 = 600.0 +# N.B. By default no delay_reaping value is configured for any accounts or +# containers. +# # recon_cache_path = /var/cache/swift # # You can set scheduling priority of processes. Niceness values range from -20 diff --git a/etc/object-server.conf-sample b/etc/object-server.conf-sample index 67c879b1ac..a95dfb9952 100644 --- a/etc/object-server.conf-sample +++ b/etc/object-server.conf-sample @@ -692,10 +692,31 @@ use = egg:swift#backend_ratelimit # ionice_class = # ionice_priority = # -# Note: Put it at the beginning of the pipleline to profile all middleware. But -# it is safer to put this after healthcheck. +# The expirer can delay the reaping of expired objects on disk (and in +# container listings) with an account level or container level delay_reaping +# time. +# After the delay_reaping time has passed objects will be reaped as normal. +# You may configure this delay_reaping value in seconds with dynamic config +# option names prefixed with delay_reaping_ for account level delays +# and delay_reaping_/ for container level delays. +# Special characters in or should be quoted. +# The delay_reaping value should be a float value greater than or equal to +# zero. +# A container level delay_reaping does not require an account level +# delay_reaping but overrides the account level delay_reaping for the same +# account if it exists. +# For example: +# delay_reaping_AUTH_test = 300.0 +# delay_reaping_AUTH_test2 = 86400.0 +# delay_reaping_AUTH_test/test = 400.0 +# delay_reaping_AUTH_test/test2 = 600.0 +# N.B. By default no delay_reaping value is configured for any accounts or +# containers. + [filter:xprofile] use = egg:swift#xprofile +# Note: Put it at the beginning of the pipleline to profile all middleware. But +# it is safer to put this after healthcheck. # This option enable you to switch profilers which should inherit from python # standard profiler. Currently the supported value can be 'cProfile', # 'eventlet.green.profile' etc. diff --git a/swift/obj/expirer.py b/swift/obj/expirer.py index c832cd63bd..6d8f864b47 100644 --- a/swift/obj/expirer.py +++ b/swift/obj/expirer.py @@ -14,6 +14,7 @@ # limitations under the License. import six +from six.moves import urllib from random import random from time import time @@ -28,7 +29,7 @@ from swift.common.daemon import Daemon from swift.common.internal_client import InternalClient, UnexpectedResponse from swift.common.utils import get_logger, dump_recon_cache, split_path, \ Timestamp, config_true_value, normalize_delete_at_timestamp, \ - RateLimitedIterator, md5 + RateLimitedIterator, md5, non_negative_float from swift.common.http import HTTP_NOT_FOUND, HTTP_CONFLICT, \ HTTP_PRECONDITION_FAILED from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH @@ -66,6 +67,49 @@ def parse_task_obj(task_obj): return timestamp, target_account, target_container, target_obj +def read_conf_for_delay_reaping_times(conf): + delay_reaping_times = {} + for conf_key in conf: + delay_reaping_prefix = "delay_reaping_" + if not conf_key.startswith(delay_reaping_prefix): + continue + delay_reaping_key = urllib.parse.unquote( + conf_key[len(delay_reaping_prefix):]) + if delay_reaping_key.strip('/') != delay_reaping_key: + raise ValueError( + '%s ' + 'should be in the form delay_reaping_ ' + 'or delay_reaping_/ ' + '(leading or trailing "/" is not allowed)' % conf_key) + try: + # If split_path fails, have multiple '/' or + # account name is invalid + account, container = split_path( + '/' + delay_reaping_key, 1, 2 + ) + except ValueError: + raise ValueError( + '%s ' + 'should be in the form delay_reaping_ ' + 'or delay_reaping_/ ' + '(at most one "/" is allowed)' % conf_key) + try: + delay_reaping_times[(account, container)] = non_negative_float( + conf.get(conf_key) + ) + except ValueError: + raise ValueError( + '%s must be a float ' + 'greater than or equal to 0' % conf_key) + return delay_reaping_times + + +def get_delay_reaping(delay_reaping_times, target_account, target_container): + return delay_reaping_times.get( + (target_account, target_container), + delay_reaping_times.get((target_account, None), 0.0)) + + class ObjectExpirer(Daemon): """ Daemon that queries the internal hidden task accounts to discover objects @@ -113,6 +157,8 @@ class ObjectExpirer(Daemon): # with the tombstone reclaim age in the consistency engine. self.reclaim_age = int(conf.get('reclaim_age', 604800)) + self.delay_reaping_times = read_conf_for_delay_reaping_times(conf) + def read_conf_for_queue_access(self, swift): self.expiring_objects_account = AUTO_CREATE_ACCOUNT_PREFIX + \ (self.conf.get('expiring_objects_account_name') or @@ -246,6 +292,10 @@ class ObjectExpirer(Daemon): break yield task_container + def get_delay_reaping(self, target_account, target_container): + return get_delay_reaping(self.delay_reaping_times, target_account, + target_container) + def iter_task_to_expire(self, task_account_container_list, my_index, divisor): """ @@ -267,17 +317,24 @@ class ObjectExpirer(Daemon): self.logger.exception('Unexcepted error handling task %r' % task_object) continue + is_async = o.get('content_type') == ASYNC_DELETE_TYPE + delay_reaping = self.get_delay_reaping(target_account, + target_container) + if delete_timestamp > Timestamp.now(): - # we shouldn't yield the object that doesn't reach + # we shouldn't yield ANY more objects that can't reach # the expiration date yet. break + if delete_timestamp > Timestamp(time() - delay_reaping) \ + and not is_async: + # we shouldn't yield the object during the delay + continue # Only one expirer daemon assigned for one task if self.hash_mod('%s/%s' % (task_container, task_object), divisor) != my_index: continue - is_async = o.get('content_type') == ASYNC_DELETE_TYPE yield {'task_account': task_account, 'task_container': task_container, 'task_object': task_object, diff --git a/test/unit/obj/test_expirer.py b/test/unit/obj/test_expirer.py index d433f8a8df..c3ef26dc9a 100644 --- a/test/unit/obj/test_expirer.py +++ b/test/unit/obj/test_expirer.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Copyright (c) 2011 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -284,6 +285,221 @@ class TestObjectExpirer(TestCase): x.get_process_values(vals) self.assertEqual(str(ctx.exception), expected_msg) + def test_valid_delay_reaping(self): + conf = {} + x = expirer.ObjectExpirer(conf, swift=self.fake_swift) + self.assertEqual(x.delay_reaping_times, {}) + + conf = { + 'delay_reaping_a': 1.0, + } + x = expirer.ObjectExpirer(conf, swift=self.fake_swift) + self.assertEqual(x.delay_reaping_times, {('a', None): 1.0}) + + # allow delay_reaping to be 0 + conf = { + 'delay_reaping_a': 0.0, + } + x = expirer.ObjectExpirer(conf, swift=self.fake_swift) + self.assertEqual(x.delay_reaping_times, {('a', None): 0.0}) + + conf = { + 'delay_reaping_a/b': 0.0, + } + x = expirer.ObjectExpirer(conf, swift=self.fake_swift) + self.assertEqual(x.delay_reaping_times, {('a', 'b'): 0.0}) + + # test configure multi-account delay_reaping + conf = { + 'delay_reaping_a': 1.0, + 'delay_reaping_b': '259200.0', + 'delay_reaping_AUTH_aBC': 999, + u'delay_reaping_AUTH_aBáC': 555, + } + x = expirer.ObjectExpirer(conf, swift=self.fake_swift) + self.assertEqual(x.delay_reaping_times, { + ('a', None): 1.0, + ('b', None): 259200.0, + ('AUTH_aBC', None): 999, + (u'AUTH_aBáC', None): 555, + }) + + # test configure multi-account delay_reaping with containers + conf = { + 'delay_reaping_a': 10.0, + 'delay_reaping_a/test': 1.0, + 'delay_reaping_b': '259200.0', + 'delay_reaping_AUTH_aBC/test2': 999, + u'delay_reaping_AUTH_aBáC/tést': 555, + 'delay_reaping_AUTH_test/special%0Achars%3Dare%20quoted': 777, + 'delay_reaping_AUTH_test/plus+signs+are+preserved': 888, + } + x = expirer.ObjectExpirer(conf, swift=self.fake_swift) + self.assertEqual(x.delay_reaping_times, { + ('a', None): 10.0, + ('a', 'test'): 1.0, + ('b', None): 259200.0, + ('AUTH_aBC', 'test2'): 999, + (u'AUTH_aBáC', u'tést'): 555, + ('AUTH_test', 'special\nchars=are quoted'): 777, + ('AUTH_test', 'plus+signs+are+preserved'): 888, + }) + + def test_invalid_delay_reaping_keys(self): + # there is no global delay_reaping + conf = { + 'delay_reaping': 0.0, + } + x = expirer.ObjectExpirer(conf, swift=self.fake_swift) + self.assertEqual(x.delay_reaping_times, {}) + + # Multiple "/" or invalid parsing + conf = { + 'delay_reaping_A_U_TH_foo_bar/my-container_name/with/slash': 60400, + } + with self.assertRaises(ValueError) as ctx: + expirer.ObjectExpirer(conf, swift=self.fake_swift) + self.assertEqual( + 'delay_reaping_A_U_TH_foo_bar/my-container_name/with/slash ' + 'should be in the form delay_reaping_ ' + 'or delay_reaping_/ ' + '(at most one "/" is allowed)', + str(ctx.exception)) + + # Can't sneak around it by escaping + conf = { + 'delay_reaping_AUTH_test/sneaky%2fsneaky': 60400, + } + with self.assertRaises(ValueError) as ctx: + expirer.ObjectExpirer(conf, swift=self.fake_swift) + self.assertEqual( + 'delay_reaping_AUTH_test/sneaky%2fsneaky ' + 'should be in the form delay_reaping_ ' + 'or delay_reaping_/ ' + '(at most one "/" is allowed)', + str(ctx.exception)) + + conf = { + 'delay_reaping_': 60400 + } + with self.assertRaises(ValueError) as ctx: + expirer.ObjectExpirer(conf, swift=self.fake_swift) + self.assertEqual( + 'delay_reaping_ ' + 'should be in the form delay_reaping_ ' + 'or delay_reaping_/ ' + '(at most one "/" is allowed)', + str(ctx.exception)) + + # Leading and trailing "/" + conf = { + 'delay_reaping_/a': 60400, + } + with self.assertRaises(ValueError) as ctx: + expirer.ObjectExpirer(conf, swift=self.fake_swift) + self.assertEqual( + 'delay_reaping_/a ' + 'should be in the form delay_reaping_ ' + 'or delay_reaping_/ ' + '(leading or trailing "/" is not allowed)', + str(ctx.exception)) + + conf = { + 'delay_reaping_a/': 60400, + } + with self.assertRaises(ValueError) as ctx: + expirer.ObjectExpirer(conf, swift=self.fake_swift) + self.assertEqual( + 'delay_reaping_a/ ' + 'should be in the form delay_reaping_ ' + 'or delay_reaping_/ ' + '(leading or trailing "/" is not allowed)', + str(ctx.exception)) + + conf = { + 'delay_reaping_/a/c/': 60400, + } + with self.assertRaises(ValueError) as ctx: + expirer.ObjectExpirer(conf, swift=self.fake_swift) + self.assertEqual( + 'delay_reaping_/a/c/ ' + 'should be in the form delay_reaping_ ' + 'or delay_reaping_/ ' + '(leading or trailing "/" is not allowed)', + str(ctx.exception)) + + def test_invalid_delay_reaping_values(self): + # negative tests + conf = { + 'delay_reaping_a': -1.0, + } + with self.assertRaises(ValueError) as ctx: + expirer.ObjectExpirer(conf, swift=self.fake_swift) + self.assertEqual( + 'delay_reaping_a must be a float greater than or equal to 0', + str(ctx.exception)) + conf = { + 'delay_reaping_a': '-259200.0' + } + with self.assertRaises(ValueError) as ctx: + expirer.ObjectExpirer(conf, swift=self.fake_swift) + self.assertEqual( + 'delay_reaping_a must be a float greater than or equal to 0', + str(ctx.exception)) + conf = { + 'delay_reaping_a': 'foo' + } + with self.assertRaises(ValueError) as ctx: + expirer.ObjectExpirer(conf, swift=self.fake_swift) + self.assertEqual( + 'delay_reaping_a must be a float greater than or equal to 0', + str(ctx.exception)) + + # negative tests with containers + conf = { + 'delay_reaping_a/b': -100.0 + } + with self.assertRaises(ValueError) as ctx: + expirer.ObjectExpirer(conf, swift=self.fake_swift) + self.assertEqual( + 'delay_reaping_a/b must be a float greater than or equal to 0', + str(ctx.exception)) + conf = { + 'delay_reaping_a/b': '-259200.0' + } + with self.assertRaises(ValueError) as ctx: + expirer.ObjectExpirer(conf, swift=self.fake_swift) + self.assertEqual( + 'delay_reaping_a/b must be a float greater than or equal to 0', + str(ctx.exception)) + conf = { + 'delay_reaping_a/b': 'foo' + } + with self.assertRaises(ValueError) as ctx: + expirer.ObjectExpirer(conf, swift=self.fake_swift) + self.assertEqual( + 'delay_reaping_a/b must be a float greater than or equal to 0', + str(ctx.exception)) + + def test_get_delay_reaping(self): + conf = { + 'delay_reaping_a': 1.0, + 'delay_reaping_a/test': 2.0, + 'delay_reaping_b': '259200.0', + 'delay_reaping_b/a': '0.0', + 'delay_reaping_c/test': '3.0' + } + x = expirer.ObjectExpirer(conf, swift=self.fake_swift) + self.assertEqual(1.0, x.get_delay_reaping('a', None)) + self.assertEqual(1.0, x.get_delay_reaping('a', 'not-test')) + self.assertEqual(2.0, x.get_delay_reaping('a', 'test')) + self.assertEqual(259200.0, x.get_delay_reaping('b', None)) + self.assertEqual(0.0, x.get_delay_reaping('b', 'a')) + self.assertEqual(259200.0, x.get_delay_reaping('b', 'test')) + self.assertEqual(3.0, x.get_delay_reaping('c', 'test')) + self.assertEqual(0.0, x.get_delay_reaping('c', 'not-test')) + self.assertEqual(0.0, x.get_delay_reaping('no-conf', 'test')) + def test_init_concurrency_too_small(self): conf = { 'concurrency': 0, @@ -746,6 +962,233 @@ class TestObjectExpirer(TestCase): task_account_container_list, my_index, divisor)), expected) + def test_iter_task_to_expire_with_delay_reaping(self): + aco_dict = { + '.expiring_objects': { + self.past_time: [ + # tasks well past ready for execution + {'name': self.past_time + '-a0/c0/o0'}, + {'name': self.past_time + '-a1/c1/o1'}, + {'name': self.past_time + '-a1/c2/o2'}, + ], + self.just_past_time: [ + # tasks only just ready for execution + {'name': self.just_past_time + '-a0/c0/o0'}, + {'name': self.just_past_time + '-a1/c1/o1'}, + {'name': self.just_past_time + '-a1/c2/o2'}, + ], + self.future_time: [ + # tasks not yet ready for execution + {'name': self.future_time + '-a0/c0/o0'}, + {'name': self.future_time + '-a1/c1/o1'}, + {'name': self.future_time + '-a1/c2/o2'}, + ], + } + } + fake_swift = FakeInternalClient(aco_dict) + # sanity, no accounts configured with delay_reaping + x = expirer.ObjectExpirer(self.conf, logger=self.logger, + swift=fake_swift) + # ... we expect tasks past time to yield + expected = [ + self.make_task(self.past_time, target_path) + for target_path in ( + swob.wsgi_to_str(tgt) for tgt in ( + 'a0/c0/o0', + 'a1/c1/o1', + 'a1/c2/o2', + ) + ) + ] + [ + self.make_task(self.just_past_time, target_path) + for target_path in ( + swob.wsgi_to_str(tgt) for tgt in ( + 'a0/c0/o0', + 'a1/c1/o1', + 'a1/c2/o2', + ) + ) + ] + task_account_container_list = [ + ('.expiring_objects', self.past_time), + ('.expiring_objects', self.just_past_time), + ] + observed = list(x.iter_task_to_expire( + task_account_container_list, 0, 1)) + self.assertEqual(expected, observed) + + # configure delay for account a1 + self.conf['delay_reaping_a1'] = 300.0 + x = expirer.ObjectExpirer(self.conf, logger=self.logger, + swift=fake_swift) + # ... and we don't expect *recent* a1 tasks or future tasks + expected = [ + self.make_task(self.past_time, target_path) + for target_path in ( + swob.wsgi_to_str(tgt) for tgt in ( + 'a0/c0/o0', + 'a1/c1/o1', + 'a1/c2/o2', + ) + ) + ] + [ + self.make_task(self.just_past_time, target_path) + for target_path in ( + swob.wsgi_to_str(tgt) for tgt in ( + 'a0/c0/o0', + ) + ) + ] + observed = list(x.iter_task_to_expire( + task_account_container_list, 0, 1)) + self.assertEqual(expected, observed) + + # configure delay for account a1 and for account a1 and container c2 + # container a1/c2 expires expires almost immediately + # but other containers in account a1 remain (a1/c1 and a1/c3) + self.conf['delay_reaping_a1'] = 300.0 + self.conf['delay_reaping_a1/c2'] = 0.1 + x = expirer.ObjectExpirer(self.conf, logger=self.logger, + swift=fake_swift) + # ... and we don't expect *recent* a1 tasks, excluding c2 + # or future tasks + expected = [ + self.make_task(self.past_time, target_path) + for target_path in ( + swob.wsgi_to_str(tgt) for tgt in ( + 'a0/c0/o0', + 'a1/c1/o1', + 'a1/c2/o2', + ) + ) + ] + [ + self.make_task(self.just_past_time, target_path) + for target_path in ( + swob.wsgi_to_str(tgt) for tgt in ( + 'a0/c0/o0', + 'a1/c2/o2', + ) + ) + ] + observed = list(x.iter_task_to_expire( + task_account_container_list, 0, 1)) + self.assertEqual(expected, observed) + + # configure delay for account a1 and for account a1 and container c2 + # container a1/c2 does not expire but others in account a1 do + self.conf['delay_reaping_a1'] = 0.1 + self.conf['delay_reaping_a1/c2'] = 300.0 + x = expirer.ObjectExpirer(self.conf, logger=self.logger, + swift=fake_swift) + # ... and we don't expect *recent* a1 tasks, excluding c2 + # or future tasks + expected = [ + self.make_task(self.past_time, target_path) + for target_path in ( + swob.wsgi_to_str(tgt) for tgt in ( + 'a0/c0/o0', + 'a1/c1/o1', + 'a1/c2/o2', + ) + ) + ] + [ + self.make_task(self.just_past_time, target_path) + for target_path in ( + swob.wsgi_to_str(tgt) for tgt in ( + 'a0/c0/o0', + 'a1/c1/o1', + ) + ) + ] + observed = list(x.iter_task_to_expire( + task_account_container_list, 0, 1)) + self.assertEqual(expected, observed) + + def test_iter_task_to_expire_with_delay_reaping_is_async(self): + aco_dict = { + '.expiring_objects': { + self.past_time: [ + # tasks well past ready for execution + {'name': self.past_time + '-a0/c0/o0', + 'content_type': 'application/async-deleted'}, + {'name': self.past_time + '-a1/c1/o1', + 'content_type': 'application/async-deleted'}, + {'name': self.past_time + '-a1/c2/o2', + 'content_type': 'application/async-deleted'}, + ], + self.just_past_time: [ + # tasks only just ready for execution + {'name': self.just_past_time + '-a0/c0/o0', + 'content_type': 'application/async-deleted'}, + {'name': self.just_past_time + '-a1/c1/o1', + 'content_type': 'application/async-deleted'}, + {'name': self.just_past_time + '-a1/c2/o2', + 'content_type': 'application/async-deleted'}, + ], + self.future_time: [ + # tasks not yet ready for execution + {'name': self.future_time + '-a0/c0/o0', + 'content_type': 'application/async-deleted'}, + {'name': self.future_time + '-a1/c1/o1', + 'content_type': 'application/async-deleted'}, + {'name': self.future_time + '-a1/c2/o2', + 'content_type': 'application/async-deleted'}, + ], + } + } + fake_swift = FakeInternalClient(aco_dict) + # no accounts configured with delay_reaping + x = expirer.ObjectExpirer(self.conf, logger=self.logger, + swift=fake_swift) + # ... we expect all past async tasks to yield + expected = [ + self.make_task(self.past_time, target_path, is_async_delete=True) + for target_path in ( + swob.wsgi_to_str(tgt) for tgt in ( + 'a0/c0/o0', + 'a1/c1/o1', + 'a1/c2/o2', + ) + ) + ] + [ + self.make_task(self.just_past_time, target_path, + is_async_delete=True) + for target_path in ( + swob.wsgi_to_str(tgt) for tgt in ( + 'a0/c0/o0', + 'a1/c1/o1', + 'a1/c2/o2', + ) + ) + ] + task_account_container_list = [ + ('.expiring_objects', self.past_time), + ('.expiring_objects', self.just_past_time), + ] + observed = list(x.iter_task_to_expire( + task_account_container_list, 0, 1)) + self.assertEqual(expected, observed) + + # configure delay for account a1 + self.conf['delay_reaping_a1'] = 300.0 + x = expirer.ObjectExpirer(self.conf, logger=self.logger, + swift=fake_swift) + # ... and we still expect all past async tasks to yield + observed = list(x.iter_task_to_expire( + task_account_container_list, 0, 1)) + self.assertEqual(expected, observed) + + # configure delays for all containers + self.conf['delay_reaping_a1/c0'] = 300.0 + self.conf['delay_reaping_a1/c1'] = 300.0 + self.conf['delay_reaping_a1/c2'] = 300.0 + x = expirer.ObjectExpirer(self.conf, logger=self.logger, + swift=fake_swift) + # ... and we we still expect all past async tasks to yield + observed = list(x.iter_task_to_expire( + task_account_container_list, 0, 1)) + self.assertEqual(expected, observed) + def test_run_once_unicode_problem(self): requests = []