diff --git a/swift/container/updater.py b/swift/container/updater.py index 5b199ae992..0f057d972c 100644 --- a/swift/container/updater.py +++ b/swift/container/updater.py @@ -29,7 +29,7 @@ import swift.common.db from swift.common.constraints import check_drive from swift.container.backend import ContainerBroker, DATADIR from swift.common.bufferedhttp import http_connect -from swift.common.exceptions import ConnectionTimeout +from swift.common.exceptions import ConnectionTimeout, LockTimeout from swift.common.ring import Ring from swift.common.utils import get_logger, config_true_value, \ dump_recon_cache, majority_size, Timestamp, ratelimit_sleep, \ @@ -232,7 +232,12 @@ class ContainerUpdater(Daemon): """ start_time = time.time() broker = ContainerBroker(dbfile, logger=self.logger) - info = broker.get_info() + try: + info = broker.get_info() + except LockTimeout: + self.logger.exception("Failed to get container info for %s", + dbfile) + return # Don't send updates if the container was auto-created since it # definitely doesn't have up to date statistics. if Timestamp(info['put_timestamp']) <= 0: diff --git a/test/unit/container/test_updater.py b/test/unit/container/test_updater.py index 9f4979bb41..8c7cadd6b3 100644 --- a/test/unit/container/test_updater.py +++ b/test/unit/container/test_updater.py @@ -25,7 +25,7 @@ from test.unit import debug_logger, mock_check_drive from eventlet import spawn, Timeout -from swift.common import utils +from swift.common import exceptions, utils from swift.container import updater as container_updater from swift.container.backend import ContainerBroker, DATADIR from swift.common.ring import RingData @@ -153,6 +153,30 @@ class TestContainerUpdater(unittest.TestCase): # Ensure that the container_sweep did not run self.assertFalse(mock_sweep.called) + @mock.patch('swift.container.updater.dump_recon_cache') + def test_run_once_with_get_info_timeout(self, mock_dump_recon): + cu = self._get_container_updater() + containers_dir = os.path.join(self.sda1, DATADIR) + os.mkdir(containers_dir) + subdir = os.path.join(containers_dir, 'subdir') + os.mkdir(subdir) + cb = ContainerBroker(os.path.join(subdir, 'hash.db'), account='a', + container='c') + cb.initialize(normalize_timestamp(1), 0) + + timeout = exceptions.LockTimeout(10) + timeout.cancel() + with mock.patch('swift.container.updater.ContainerBroker.get_info', + side_effect=timeout): + cu.run_once() + log_lines = self.logger.get_lines_for_level('error') + self.assertTrue(log_lines) + self.assertIn('Failed to get container info ', log_lines[0]) + self.assertIn('devices/sda1/containers/subdir/hash.db', log_lines[0]) + self.assertIn('LockTimeout (10s)', log_lines[0]) + self.assertFalse(log_lines[1:]) + self.assertEqual(1, len(mock_dump_recon.mock_calls)) + def test_run_once(self): cu = self._get_container_updater() cu.run_once()