openstack/swift
Code Issues Proposed changes

Merge "proxy-server: fix node error limiting"

Browse Source
This commit is contained in:
Zuul 2023-09-27 20:17:51 +00:00 committed by Gerrit Code Review
commit d25bec42fb
2 changed files with 35 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
swift/proxy/controllers/base.py
View File

@ -1125,18 +1125,15 @@ class GetterBase(object):
:return: ``True`` if ``self.source`` has been updated, ``False`` :return: ``True`` if ``self.source`` has been updated, ``False``
otherwise. otherwise.
""" """
# Subclasses must implement this method # Subclasses must implement this method, but _replace_source should be
# called to get a source installed
raise NotImplementedError() raise NotImplementedError()
def _replace_source(self, err_msg): def _replace_source(self, err_msg=''):
# _find_source can modify self.source so stash current source if self.source:
old_source = self.source self.app.error_occurred(self.source.node, err_msg)
if not self._find_source(): self.source.close()
return False return self._find_source()
self.app.error_occurred(old_source.node, err_msg)
old_source.close()
return True
def fast_forward(self, num_bytes): def fast_forward(self, num_bytes):
""" """
@ -1613,7 +1610,7 @@ class GetOrHeadHandler(GetterBase):
def get_working_response(self, req): def get_working_response(self, req):
res = None res = None
if self._find_source(): if self._replace_source():
res = Response(request=req) res = Response(request=req)
res.status = self.source.resp.status res.status = self.source.resp.status
update_headers(res, self.source.resp.getheaders()) update_headers(res, self.source.resp.getheaders())

41
test/unit/proxy/controllers/test_obj.py
View File

@ -4593,12 +4593,14 @@ class TestECObjController(ECObjectControllerMixin, unittest.TestCase):
self.assertEqual(resp.status_int, 500) self.assertEqual(resp.status_int, 500)
self.assertEqual(len(log), self.policy.ec_n_unique_fragments * 2) self.assertEqual(len(log), self.policy.ec_n_unique_fragments * 2)
log_lines = self.app.logger.get_lines_for_level('error') log_lines = self.app.logger.get_lines_for_level('error')
self.assertEqual(2, len(log_lines), log_lines) self.assertEqual(3, len(log_lines), log_lines)
self.assertIn('Trying to read during GET: ChunkReadTimeout', self.assertIn('Trying to read next part of EC multi-part GET',
log_lines[0]) log_lines[0])
self.assertIn('Trying to read during GET: ChunkReadTimeout',
log_lines[1])
# not the most graceful ending # not the most graceful ending
self.assertIn('Unhandled exception in request: ChunkReadTimeout', self.assertIn('Unhandled exception in request: ChunkReadTimeout',
log_lines[1]) log_lines[2])
def test_GET_with_multirange_short_resume_body(self): def test_GET_with_multirange_short_resume_body(self):
self.app.object_chunk_size = 256 self.app.object_chunk_size = 256
@ -5137,12 +5139,17 @@ class TestECObjController(ECObjectControllerMixin, unittest.TestCase):
md5(resp.body, usedforsecurity=False).hexdigest(), md5(resp.body, usedforsecurity=False).hexdigest(),
etag) etag)
error_lines = self.logger.get_lines_for_level('error') error_lines = self.logger.get_lines_for_level('error')
nparity = self.policy.ec_nparity # all primaries timeout and get error limited
self.assertGreater(len(error_lines), nparity) error_limit_lines = [
for line in error_lines[:nparity]: line for line in error_lines
self.assertIn('retrying', line) if 'Trying to read EC fragment during GET (retrying)' in line]
for line in error_lines[nparity:]: self.assertEqual(self.policy.ec_n_unique_fragments,
self.assertIn('ChunkReadTimeout (0.01s', line) len(error_limit_lines))
# all ec_ndata frag getters eventually get a read timeout
read_timeout_lines = [
line for line in error_lines if 'ChunkReadTimeout (0.01s' in line]
self.assertEqual(self.policy.ec_ndata,
len(read_timeout_lines))
for line in self.logger.logger.records['ERROR']: for line in self.logger.logger.records['ERROR']:
self.assertIn(req.headers['x-trans-id'], line) self.assertIn(req.headers['x-trans-id'], line)
@ -5225,11 +5232,17 @@ class TestECObjController(ECObjectControllerMixin, unittest.TestCase):
# resume but won't be able to give us all the right bytes # resume but won't be able to give us all the right bytes
self.assertNotEqual(md5(resp.body).hexdigest(), etag) self.assertNotEqual(md5(resp.body).hexdigest(), etag)
error_lines = self.logger.get_lines_for_level('error') error_lines = self.logger.get_lines_for_level('error')
self.assertEqual(ndata, len(error_lines)) # only ec_ndata primaries that timeout get error limited (404 or
for line in error_lines: # different etag primaries do not get error limited)
self.assertIn('ChunkReadTimeout (0.01s', line) error_limit_lines = [
for line in self.logger.logger.records['ERROR']: line for line in error_lines
self.assertIn(req.headers['x-trans-id'], line) if 'Trying to read EC fragment during GET (retrying)' in line]
self.assertEqual(self.policy.ec_ndata, len(error_limit_lines))
# all ec_ndata frag getters eventually get a read timeout
read_timeout_lines = [
line for line in error_lines if 'ChunkReadTimeout (0.01s' in line]
self.assertEqual(self.policy.ec_ndata,
len(read_timeout_lines))
debug_lines = self.logger.get_lines_for_level('debug') debug_lines = self.logger.get_lines_for_level('debug')
nparity = self.policy.ec_nparity nparity = self.policy.ec_nparity