From 44ba3c310a90f6db49417220ca979fd945ca1799 Mon Sep 17 00:00:00 2001 From: Doron Chen Date: Wed, 10 Aug 2016 09:29:44 +0300 Subject: [PATCH] Added a test for delayed reap. The test schedules a 3-second delayed account reaping. The test checks that no reaping after an immediate reap, and that full reaping occurs 3 seconds later. Change-Id: I0ab954ed3c59d808f32d84dc53fd512fd0a651be --- test/probe/test_account_reaper.py | 75 +++++++++++++++++++++++++------ 1 file changed, 62 insertions(+), 13 deletions(-) diff --git a/test/probe/test_account_reaper.py b/test/probe/test_account_reaper.py index f5d2efee87..bc83d7716b 100644 --- a/test/probe/test_account_reaper.py +++ b/test/probe/test_account_reaper.py @@ -12,12 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from time import sleep import uuid import unittest from swiftclient import client -from swift.common.storage_policy import POLICIES +from swift.account import reaper +from swift.common import utils from swift.common.manager import Manager from swift.common.direct_client import direct_delete_account, \ direct_get_object, direct_head_container, ClientException @@ -25,18 +27,19 @@ from test.probe.common import ReplProbeTest, ENABLED_POLICIES class TestAccountReaper(ReplProbeTest): - - def test_sync(self): - all_objects = [] + def setUp(self): + super(TestAccountReaper, self).setUp() + self.all_objects = [] # upload some containers + body = 'test-body' for policy in ENABLED_POLICIES: container = 'container-%s-%s' % (policy.name, uuid.uuid4()) client.put_container(self.url, self.token, container, headers={'X-Storage-Policy': policy.name}) obj = 'object-%s' % uuid.uuid4() - body = 'test-body' client.put_object(self.url, self.token, container, obj, body) - all_objects.append((policy, container, obj)) + self.all_objects.append((policy, container, obj)) + policy.load_ring('/etc/swift') Manager(['container-updater']).once() @@ -50,13 +53,18 @@ class TestAccountReaper(ReplProbeTest): len(ENABLED_POLICIES) * len(body)) part, nodes = self.account_ring.get_nodes(self.account) + for node in nodes: direct_delete_account(node, part, self.account) + def test_sync(self): # run the reaper Manager(['account-reaper']).once() - for policy, container, obj in all_objects: + self._verify_account_reaped() + + def _verify_account_reaped(self): + for policy, container, obj in self.all_objects: # verify that any container deletes were at same timestamp cpart, cnodes = self.container_ring.get_nodes( self.account, container) @@ -72,7 +80,6 @@ class TestAccountReaper(ReplProbeTest): # 'X-Backend-DELETE-Timestamp' confirms it was deleted self.assertTrue(delete_time) delete_times.add(delete_time) - else: # Container replicas may not yet be deleted if we have a # policy with object replicas < container replicas, so @@ -82,8 +89,8 @@ class TestAccountReaper(ReplProbeTest): self.assertEqual(1, len(delete_times), delete_times) # verify that all object deletes were at same timestamp - object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/') - part, nodes = object_ring.get_nodes(self.account, container, obj) + part, nodes = policy.object_ring.get_nodes(self.account, + container, obj) headers = {'X-Backend-Storage-Policy-Index': int(policy)} delete_times = set() for node in nodes: @@ -104,7 +111,7 @@ class TestAccountReaper(ReplProbeTest): # run replicators and updaters self.get_to_final_state() - for policy, container, obj in all_objects: + for policy, container, obj in self.all_objects: # verify that ALL container replicas are now deleted cpart, cnodes = self.container_ring.get_nodes( self.account, container) @@ -123,10 +130,11 @@ class TestAccountReaper(ReplProbeTest): else: self.fail('Found un-reaped /%s/%s on %r' % (self.account, container, cnode)) + self.assertEqual(1, len(delete_times)) # sanity check that object state is still consistent... - object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/') - part, nodes = object_ring.get_nodes(self.account, container, obj) + part, nodes = policy.object_ring.get_nodes(self.account, + container, obj) headers = {'X-Backend-Storage-Policy-Index': int(policy)} delete_times = set() for node in nodes: @@ -144,6 +152,47 @@ class TestAccountReaper(ReplProbeTest): (self.account, container, obj, node, policy)) self.assertEqual(1, len(delete_times)) + def test_delayed_reap(self): + # define reapers which are supposed to operate 3 seconds later + account_reapers = [] + for conf_file in self.configs['account-server'].values(): + conf = utils.readconf(conf_file, 'account-reaper') + conf['delay_reaping'] = '3' + account_reapers.append(reaper.AccountReaper(conf)) + + self.assertTrue(account_reapers) + + # run reaper, and make sure that nothing is reaped + for account_reaper in account_reapers: + account_reaper.run_once() + + for policy, container, obj in self.all_objects: + cpart, cnodes = self.container_ring.get_nodes( + self.account, container) + for cnode in cnodes: + try: + direct_head_container(cnode, cpart, self.account, + container) + except ClientException: + self.fail("Nothing should be reaped. Object should exist") + + part, nodes = policy.object_ring.get_nodes(self.account, + container, obj) + headers = {'X-Backend-Storage-Policy-Index': int(policy)} + for node in nodes: + try: + direct_get_object(node, part, self.account, + container, obj, headers=headers) + except ClientException: + self.fail("Nothing should be reaped. Object should exist") + + # wait 3 seconds, run reaper, and make sure that all is reaped + sleep(3) + for account_reaper in account_reapers: + account_reaper.run_once() + + self._verify_account_reaped() + if __name__ == "__main__": unittest.main()