openstack/swift
Code Issues Proposed changes

Catch and log errors from process_container, then keep updating

Browse Source 
Previously, we'd bomb out hard, and you'd get no more container updates.

Change-Id: I0cea6018269d172bc758eec6348329d1f65de028
Related-Bug: 1400939
This commit is contained in:
Tim Burke 2018-05-25 16:54:03 -07:00
parent 4189a117d2
commit 31623f740b
2 changed files with 29 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
swift/container/updater.py
View File

@ -218,7 +218,12 @@ class ContainerUpdater(Daemon):
for root, dirs, files in os.walk(path):
for file in files:
if file.endswith('.db'):
self.process_container(os.path.join(root, file))
dbfile = os.path.join(root, file)
try:
self.process_container(dbfile)
except (Exception, Timeout) as e:
self.logger.exception(
"Error processing container %s: %s", dbfile, e)
self.containers_running_time = ratelimit_sleep(
self.containers_running_time,

23
test/unit/container/test_updater.py
View File

@ -177,6 +177,29 @@ class TestContainerUpdater(unittest.TestCase):
self.assertFalse(log_lines[1:])
self.assertEqual(1, len(mock_dump_recon.mock_calls))
@mock.patch('swift.container.updater.dump_recon_cache')
@mock.patch('swift.container.updater.ContainerUpdater.process_container',
side_effect=Exception('Boom!'))
def test_error_in_process(self, mock_process, mock_dump_recon):
cu = self._get_container_updater()
containers_dir = os.path.join(self.sda1, DATADIR)
os.mkdir(containers_dir)
subdir = os.path.join(containers_dir, 'subdir')
os.mkdir(subdir)
cb = ContainerBroker(os.path.join(subdir, 'hash.db'), account='a',
container='c', pending_timeout=1)
cb.initialize(normalize_timestamp(1), 0)
cu.run_once()
log_lines = self.logger.get_lines_for_level('error')
self.assertTrue(log_lines)
self.assertIn('Error processing container ', log_lines[0])
self.assertIn('devices/sda1/containers/subdir/hash.db', log_lines[0])
self.assertIn('Boom!', log_lines[0])
self.assertFalse(log_lines[1:])
self.assertEqual(1, len(mock_dump_recon.mock_calls))
def test_run_once(self):
cu = self._get_container_updater()
cu.run_once()