From 4189a117d27f92afa5ab983689fb6b781b93ef02 Mon Sep 17 00:00:00 2001 From: Hu Bing Date: Fri, 18 Dec 2015 12:50:41 +0800 Subject: [PATCH] catch lock fail exception in container updater in process of container update, when locking one container failed, container updater stops with LockTimeout exception. it's better for updater server to continue processing other containers, partitions instead of stop updating. this path is to catch timeout exception and log it, then continue processing other containers and partitions. Closes-bug: #1400939 Change-Id: I42adec07d980be22044a5d4ef6771318a3eed168 --- swift/container/updater.py | 9 +++++++-- test/unit/container/test_updater.py | 26 +++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/swift/container/updater.py b/swift/container/updater.py index 5b199ae992..0f057d972c 100644 --- a/swift/container/updater.py +++ b/swift/container/updater.py @@ -29,7 +29,7 @@ import swift.common.db from swift.common.constraints import check_drive from swift.container.backend import ContainerBroker, DATADIR from swift.common.bufferedhttp import http_connect -from swift.common.exceptions import ConnectionTimeout +from swift.common.exceptions import ConnectionTimeout, LockTimeout from swift.common.ring import Ring from swift.common.utils import get_logger, config_true_value, \ dump_recon_cache, majority_size, Timestamp, ratelimit_sleep, \ @@ -232,7 +232,12 @@ class ContainerUpdater(Daemon): """ start_time = time.time() broker = ContainerBroker(dbfile, logger=self.logger) - info = broker.get_info() + try: + info = broker.get_info() + except LockTimeout: + self.logger.exception("Failed to get container info for %s", + dbfile) + return # Don't send updates if the container was auto-created since it # definitely doesn't have up to date statistics. if Timestamp(info['put_timestamp']) <= 0: diff --git a/test/unit/container/test_updater.py b/test/unit/container/test_updater.py index 9f4979bb41..8c7cadd6b3 100644 --- a/test/unit/container/test_updater.py +++ b/test/unit/container/test_updater.py @@ -25,7 +25,7 @@ from test.unit import debug_logger, mock_check_drive from eventlet import spawn, Timeout -from swift.common import utils +from swift.common import exceptions, utils from swift.container import updater as container_updater from swift.container.backend import ContainerBroker, DATADIR from swift.common.ring import RingData @@ -153,6 +153,30 @@ class TestContainerUpdater(unittest.TestCase): # Ensure that the container_sweep did not run self.assertFalse(mock_sweep.called) + @mock.patch('swift.container.updater.dump_recon_cache') + def test_run_once_with_get_info_timeout(self, mock_dump_recon): + cu = self._get_container_updater() + containers_dir = os.path.join(self.sda1, DATADIR) + os.mkdir(containers_dir) + subdir = os.path.join(containers_dir, 'subdir') + os.mkdir(subdir) + cb = ContainerBroker(os.path.join(subdir, 'hash.db'), account='a', + container='c') + cb.initialize(normalize_timestamp(1), 0) + + timeout = exceptions.LockTimeout(10) + timeout.cancel() + with mock.patch('swift.container.updater.ContainerBroker.get_info', + side_effect=timeout): + cu.run_once() + log_lines = self.logger.get_lines_for_level('error') + self.assertTrue(log_lines) + self.assertIn('Failed to get container info ', log_lines[0]) + self.assertIn('devices/sda1/containers/subdir/hash.db', log_lines[0]) + self.assertIn('LockTimeout (10s)', log_lines[0]) + self.assertFalse(log_lines[1:]) + self.assertEqual(1, len(mock_dump_recon.mock_calls)) + def test_run_once(self): cu = self._get_container_updater() cu.run_once()