Merge "catch lock fail exception in container updater"

swift/container/updater.py

@@ -29,7 +29,7 @@ import swift.common.db
 from swift.common.constraints import check_drive
 from swift.container.backend import ContainerBroker, DATADIR
 from swift.common.bufferedhttp import http_connect
-from swift.common.exceptions import ConnectionTimeout
+from swift.common.exceptions import ConnectionTimeout, LockTimeout
 from swift.common.ring import Ring
 from swift.common.utils import get_logger, config_true_value, \
     dump_recon_cache, majority_size, Timestamp, ratelimit_sleep, \
@@ -232,7 +232,12 @@ class ContainerUpdater(Daemon):
         """
         start_time = time.time()
         broker = ContainerBroker(dbfile, logger=self.logger)
-        info = broker.get_info()
+        try:
+            info = broker.get_info()
+        except LockTimeout:
+            self.logger.exception("Failed to get container info for %s",
+                                  dbfile)
+            return
         # Don't send updates if the container was auto-created since it
         # definitely doesn't have up to date statistics.
         if Timestamp(info['put_timestamp']) <= 0:

test/unit/container/test_updater.py

@@ -25,7 +25,7 @@ from test.unit import debug_logger, mock_check_drive
 
 from eventlet import spawn, Timeout
 
-from swift.common import utils
+from swift.common import exceptions, utils
 from swift.container import updater as container_updater
 from swift.container.backend import ContainerBroker, DATADIR
 from swift.common.ring import RingData
@@ -153,6 +153,30 @@ class TestContainerUpdater(unittest.TestCase):
         # Ensure that the container_sweep did not run
         self.assertFalse(mock_sweep.called)
 
+    @mock.patch('swift.container.updater.dump_recon_cache')
+    def test_run_once_with_get_info_timeout(self, mock_dump_recon):
+        cu = self._get_container_updater()
+        containers_dir = os.path.join(self.sda1, DATADIR)
+        os.mkdir(containers_dir)
+        subdir = os.path.join(containers_dir, 'subdir')
+        os.mkdir(subdir)
+        cb = ContainerBroker(os.path.join(subdir, 'hash.db'), account='a',
+                             container='c')
+        cb.initialize(normalize_timestamp(1), 0)
+
+        timeout = exceptions.LockTimeout(10)
+        timeout.cancel()
+        with mock.patch('swift.container.updater.ContainerBroker.get_info',
+                        side_effect=timeout):
+            cu.run_once()
+        log_lines = self.logger.get_lines_for_level('error')
+        self.assertTrue(log_lines)
+        self.assertIn('Failed to get container info ', log_lines[0])
+        self.assertIn('devices/sda1/containers/subdir/hash.db', log_lines[0])
+        self.assertIn('LockTimeout (10s)', log_lines[0])
+        self.assertFalse(log_lines[1:])
+        self.assertEqual(1, len(mock_dump_recon.mock_calls))
+
     def test_run_once(self):
         cu = self._get_container_updater()
         cu.run_once()