openstack/swift
Code Issues Proposed changes

Merge "proxy: refactor error limiter to a class"

Browse Source
This commit is contained in:
Zuul 2022-11-02 18:15:08 +00:00 committed by Gerrit Code Review
commit ed751118ba
6 changed files with 289 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
swift/common/error_limiter.py Normal file
View File

@ -0,0 +1,91 @@
# Copyright (c) 2021 NVIDIA
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import collections
from time import time
from swift.common.utils import node_to_string
class ErrorLimiter(object):
"""
Tracks the number of errors that have occurred for nodes. A node will be
considered to be error-limited for a given interval of time after it has
accumulated more errors than a given limit.
:param suppression_interval: The number of seconds for which a node is
error-limited once it has accumulated more than ``suppression_limit``
errors. Should be a float value.
:param suppression_limit: The number of errors that a node must accumulate
before it is considered to be error-limited. Should be an int value.
"""
def __init__(self, suppression_interval, suppression_limit):
self.suppression_interval = float(suppression_interval)
self.suppression_limit = int(suppression_limit)
self.stats = collections.defaultdict(dict)
def node_key(self, node):
"""
Get the key under which a node's error stats will be stored.
:param node: dictionary describing a node.
:return: string key.
"""
return node_to_string(node)
def is_limited(self, node):
"""
Check if the node is currently error limited.
:param node: dictionary of node to check
:returns: True if error limited, False otherwise
"""
now = time()
node_key = self.node_key(node)
error_stats = self.stats.get(node_key)
if error_stats is None or 'errors' not in error_stats:
return False
if 'last_error' in error_stats and error_stats['last_error'] < \
now - self.suppression_interval:
self.stats.pop(node_key)
return False
return error_stats['errors'] > self.suppression_limit
def limit(self, node):
"""
Mark a node as error limited. This immediately pretends the
node received enough errors to trigger error suppression. Use
this for errors like Insufficient Storage. For other errors
use :func:`increment`.
:param node: dictionary of node to error limit
"""
node_key = self.node_key(node)
error_stats = self.stats[node_key]
error_stats['errors'] = self.suppression_limit + 1
error_stats['last_error'] = time()
def increment(self, node):
"""
Increment the error count and update the time of the last error for
the given ``node``.
:param node: dictionary describing a node.
"""
node_key = self.node_key(node)
error_stats = self.stats[node_key]
error_stats['errors'] = error_stats.get('errors', 0) + 1
error_stats['last_error'] = time()

40
swift/proxy/server.py
View File

@ -31,6 +31,7 @@ from swift.common import constraints
from swift.common.http import is_server_error
from swift.common.storage_policy import POLICIES
from swift.common.ring import Ring
from swift.common.error_limiter import ErrorLimiter
from swift.common.utils import Watchdog, get_logger, \
get_remote_client, split_path, config_true_value, generate_trans_id, \
affinity_key_function, affinity_locality_predicate, list_from_csv, \
@ -204,7 +205,6 @@ class Application(object):
statsd_tail_prefix='proxy-server')
else:
self.logger = logger
self._error_limiting = {}
self.backend_user_agent = 'proxy-server %s' % os.getpid()
swift_dir = conf.get('swift_dir', '/etc/swift')
@ -218,10 +218,12 @@ class Application(object):
self.client_chunk_size = int(conf.get('client_chunk_size', 65536))
self.trans_id_suffix = conf.get('trans_id_suffix', '')
self.post_quorum_timeout = float(conf.get('post_quorum_timeout', 0.5))
self.error_suppression_interval = \
error_suppression_interval = \
float(conf.get('error_suppression_interval', 60))
self.error_suppression_limit = \
error_suppression_limit = \
int(conf.get('error_suppression_limit', 10))
self.error_limiter = ErrorLimiter(error_suppression_interval,
error_suppression_limit)
self.recheck_container_existence = \
int(conf.get('recheck_container_existence',
DEFAULT_RECHECK_CONTAINER_EXISTENCE))
@ -646,9 +648,6 @@ class Application(object):
timing = round(timing, 3) # sort timings to the millisecond
self.node_timings[node['ip']] = (timing, now + self.timing_expiry)
def _error_limit_node_key(self, node):
return node_to_string(node)
def error_limited(self, node):
"""
Check if the node is currently error limited.
@ -656,17 +655,7 @@ class Application(object):
:param node: dictionary of node to check
:returns: True if error limited, False otherwise
"""
now = time()
node_key = self._error_limit_node_key(node)
error_stats = self._error_limiting.get(node_key)
if error_stats is None or 'errors' not in error_stats:
return False
if 'last_error' in error_stats and error_stats['last_error'] < \
now - self.error_suppression_interval:
self._error_limiting.pop(node_key, None)
return False
limited = error_stats['errors'] > self.error_suppression_limit
limited = self.error_limiter.is_limited(node)
if limited:
self.logger.debug(
'Node error limited: %s', node_to_string(node))
@ -677,24 +666,15 @@ class Application(object):
Mark a node as error limited. This immediately pretends the
node received enough errors to trigger error suppression. Use
this for errors like Insufficient Storage. For other errors
use :func:`error_occurred`.
use :func:`increment`.
:param node: dictionary of node to error limit
:param msg: error message
"""
node_key = self._error_limit_node_key(node)
error_stats = self._error_limiting.setdefault(node_key, {})
error_stats['errors'] = self.error_suppression_limit + 1
error_stats['last_error'] = time()
self.error_limiter.limit(node)
self.logger.error('%(msg)s %(node)s',
{'msg': msg, 'node': node_to_string(node)})
def _incr_node_errors(self, node):
node_key = self._error_limit_node_key(node)
error_stats = self._error_limiting.setdefault(node_key, {})
error_stats['errors'] = error_stats.get('errors', 0) + 1
error_stats['last_error'] = time()
def error_occurred(self, node, msg):
"""
Handle logging, and handling of errors.
@ -702,7 +682,7 @@ class Application(object):
:param node: dictionary of node to handle errors for
:param msg: error message
"""
self._incr_node_errors(node)
self.error_limiter.increment(node)
if isinstance(msg, bytes):
msg = msg.decode('utf-8')
self.logger.error('%(msg)s %(node)s',
@ -721,7 +701,7 @@ class Application(object):
:param typ: server type
:param additional_info: additional information to log
"""
self._incr_node_errors(node)
self.error_limiter.increment(node)
if 'level' in kwargs:
log = functools.partial(self.logger.log, kwargs.pop('level'))
if 'exc_info' not in kwargs:

102
test/unit/common/test_error_limiter.py Normal file
View File

@ -0,0 +1,102 @@
# Copyright (c) 2021 NVIDIA
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import mock
from time import time
from swift.common.error_limiter import ErrorLimiter
from test.unit import FakeRing
class TestErrorLimiter(unittest.TestCase):
def setUp(self):
self.ring = FakeRing()
def test_init_config(self):
config = {'suppression_interval': 100.9,
'suppression_limit': 5}
limiter = ErrorLimiter(**config)
self.assertEqual(limiter.suppression_interval, 100.9)
self.assertEqual(limiter.suppression_limit, 5)
config = {'suppression_interval': '100.9',
'suppression_limit': '5'}
limiter = ErrorLimiter(**config)
self.assertEqual(limiter.suppression_interval, 100.9)
self.assertEqual(limiter.suppression_limit, 5)
def test_init_bad_config(self):
with self.assertRaises(ValueError):
ErrorLimiter(suppression_interval='bad',
suppression_limit=1)
with self.assertRaises(TypeError):
ErrorLimiter(suppression_interval=None,
suppression_limit=1)
with self.assertRaises(ValueError):
ErrorLimiter(suppression_interval=0,
suppression_limit='bad')
with self.assertRaises(TypeError):
ErrorLimiter(suppression_interval=0,
suppression_limit=None)
def test_is_limited(self):
node = self.ring.devs[-1]
limiter = ErrorLimiter(suppression_interval=60, suppression_limit=10)
now = time()
with mock.patch('swift.common.error_limiter.time', return_value=now):
self.assertFalse(limiter.is_limited(node))
limiter.limit(node)
self.assertTrue(limiter.is_limited(node))
node_key = limiter.node_key(node)
self.assertEqual(limiter.stats.get(node_key),
{'errors': limiter.suppression_limit + 1,
'last_error': now})
def test_increment(self):
node = self.ring.devs[-1]
limiter = ErrorLimiter(suppression_interval=60, suppression_limit=10)
last_errors = 0
node_key = limiter.node_key(node)
for _ in range(limiter.suppression_limit):
limiter.increment(node)
node_errors = limiter.stats.get(node_key)
self.assertGreater(node_errors['errors'], last_errors)
self.assertFalse(limiter.is_limited(node))
last_errors = node_errors['errors']
# One more to make sure it is > suppression_limit
limiter.increment(node)
node_errors = limiter.stats.get(node_key)
self.assertEqual(limiter.suppression_limit + 1,
node_errors['errors'])
self.assertTrue(limiter.is_limited(node))
last_time = node_errors['last_error']
# Simulate time with no errors have gone by.
now = last_time + limiter.suppression_interval + 1
with mock.patch('swift.common.error_limiter.time',
return_value=now):
self.assertFalse(limiter.is_limited(node))
self.assertFalse(limiter.stats.get(node_key))
def test_node_key(self):
limiter = ErrorLimiter(suppression_interval=60, suppression_limit=10)
node = self.ring.devs[0]
expected = '%s:%s/%s' % (node['ip'], node['port'], node['device'])
self.assertEqual(expected, limiter.node_key(node))

4
test/unit/proxy/controllers/test_container.py
View File

@ -318,7 +318,7 @@ class TestContainerController(TestRingBase):
for method in ('PUT', 'DELETE', 'POST'):
def test_status_map(statuses, expected):
self.app._error_limiting = {}
self.app.error_limiter.stats.clear()
req = Request.blank('/v1/a/c', method=method)
with mocked_http_conn(*statuses) as fake_conn:
resp = req.get_response(self.app)
@ -355,7 +355,7 @@ class TestContainerController(TestRingBase):
test_status_map(base_status[:2] + [507] + base_status[2:], 201)
self.assertEqual(node_error_count(
self.app, self.container_ring.devs[2]),
self.app.error_suppression_limit + 1)
self.app.error_limiter.suppression_limit + 1)
def test_response_codes_for_GET(self):
nodes = self.app.container_ring.replicas

18
test/unit/proxy/controllers/test_obj.py
View File

@ -1214,7 +1214,7 @@ class TestReplicatedObjController(CommonObjectControllerMixin,
self.app.sort_nodes = lambda n, *args, **kwargs: n # disable shuffle
def test_status_map(statuses, expected):
self.app._error_limiting = {}
self.app.error_limiter.stats.clear()
req = swob.Request.blank('/v1/a/c/o.jpg', method='PUT',
body=b'test body')
with set_http_connect(*statuses):
@ -1249,7 +1249,7 @@ class TestReplicatedObjController(CommonObjectControllerMixin,
test_status_map(((507, None), 201, 201, 201), 201)
self.assertEqual(
node_error_count(self.app, object_ring.devs[0]),
self.app.error_suppression_limit + 1)
self.app.error_limiter.suppression_limit + 1)
# response errors
test_status_map(((100, Timeout()), 201, 201), 201)
self.assertEqual(
@ -1260,7 +1260,7 @@ class TestReplicatedObjController(CommonObjectControllerMixin,
test_status_map((201, (100, 507), 201), 201)
self.assertEqual(
node_error_count(self.app, object_ring.devs[1]),
self.app.error_suppression_limit + 1)
self.app.error_limiter.suppression_limit + 1)
def test_PUT_connect_exception_with_unicode_path(self):
expected = 201
@ -2385,7 +2385,13 @@ class TestECObjController(ECObjectControllerMixin, unittest.TestCase):
}
for r in log.requests[:4]
}
self.assertEqual(self.app._error_limiting, expected_error_limiting)
actual = {}
for n in self.app.get_object_ring(int(self.policy)).devs:
node_key = self.app.error_limiter.node_key(n)
stats = self.app.error_limiter.stats.get(node_key) or {}
if stats:
actual[self.app.error_limiter.node_key(n)] = stats
self.assertEqual(actual, expected_error_limiting)
def test_GET_not_found_when_404_newer(self):
# if proxy receives a 404, it keeps waiting for other connections until
@ -2410,14 +2416,14 @@ class TestECObjController(ECObjectControllerMixin, unittest.TestCase):
with mocked_http_conn(*codes):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 404)
self.app._error_limiting = {} # Reset error limiting
self.app.error_limiter.stats.clear() # Reset error limiting
# one more timeout is past the tipping point
codes[self.policy.object_ring.replica_count - 2] = Timeout()
with mocked_http_conn(*codes):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 503)
self.app._error_limiting = {} # Reset error limiting
self.app.error_limiter.stats.clear() # Reset error limiting
# unless we have tombstones
with mocked_http_conn(*codes, headers={'X-Backend-Timestamp': '1'}):

140
test/unit/proxy/test_server.py
View File

@ -150,26 +150,29 @@ def parse_headers_string(headers_str):
return headers_dict
def get_node_error_stats(proxy_app, ring_node):
node_key = proxy_app.error_limiter.node_key(ring_node)
return proxy_app.error_limiter.stats.get(node_key) or {}
def node_error_count(proxy_app, ring_node):
# Reach into the proxy's internals to get the error count for a
# particular node
node_key = proxy_app._error_limit_node_key(ring_node)
return proxy_app._error_limiting.get(node_key, {}).get('errors', 0)
return get_node_error_stats(proxy_app, ring_node).get('errors', 0)
def node_last_error(proxy_app, ring_node):
# Reach into the proxy's internals to get the last error for a
# particular node
node_key = proxy_app._error_limit_node_key(ring_node)
return proxy_app._error_limiting.get(node_key, {}).get('last_error')
return get_node_error_stats(proxy_app, ring_node).get('last_error')
def set_node_errors(proxy_app, ring_node, value, last_error):
# Set the node's error count to value
node_key = proxy_app._error_limit_node_key(ring_node)
stats = proxy_app._error_limiting.setdefault(node_key, {})
stats['errors'] = value
stats['last_error'] = last_error
node_key = proxy_app.error_limiter.node_key(ring_node)
stats = {'errors': value,
'last_error': last_error}
proxy_app.error_limiter.stats[node_key] = stats
@contextmanager
@ -239,7 +242,7 @@ class TestController(unittest.TestCase):
def setUp(self):
skip_if_no_xattrs()
_test_servers[0]._error_limiting = {} # clear out errors
_test_servers[0].error_limiter.stats.clear() # clear out errors
self.account_ring = FakeRing()
self.container_ring = FakeRing()
self.memcache = FakeMemcache()
@ -1169,15 +1172,15 @@ class TestProxyServer(unittest.TestCase):
container_ring=FakeRing(),
logger=logger)
node = app.container_ring.get_part_nodes(0)[0]
node_key = app._error_limit_node_key(node)
self.assertNotIn(node_key, app._error_limiting) # sanity
node_key = app.error_limiter.node_key(node)
self.assertNotIn(node_key, app.error_limiter.stats) # sanity
try:
raise Exception('kaboom1!')
except Exception as err:
caught_exc = err
app.exception_occurred(node, 'server-type', additional_info)
self.assertEqual(1, app._error_limiting[node_key]['errors'])
self.assertEqual(1, node_error_count(app, node))
line = logger.get_lines_for_level('error')[-1]
self.assertIn('server-type server', line)
if six.PY2:
@ -1203,12 +1206,12 @@ class TestProxyServer(unittest.TestCase):
container_ring=FakeRing(),
logger=logger)
node = app.container_ring.get_part_nodes(0)[0]
node_key = app._error_limit_node_key(node)
self.assertNotIn(node_key, app._error_limiting) # sanity
node_key = app.error_limiter.node_key(node)
self.assertNotIn(node_key, app.error_limiter.stats) # sanity
app.error_occurred(node, msg)
self.assertEqual(1, app._error_limiting[node_key]['errors'])
self.assertEqual(1, node_error_count(app, node))
line = logger.get_lines_for_level('error')[-1]
if six.PY2:
self.assertIn(msg.decode('utf8'), line)
@ -1501,7 +1504,7 @@ class TestProxyServerConfigLoading(unittest.TestCase):
def setUp(self):
skip_if_no_xattrs()
_test_servers[0]._error_limiting = {} # clear out errors
_test_servers[0].error_limiter.stats.clear() # clear out errors
self.tempdir = mkdtemp()
account_ring_path = os.path.join(self.tempdir, 'account.ring.gz')
write_fake_ring(account_ring_path)
@ -2362,7 +2365,7 @@ class TestReplicatedObjectController(
"""
def setUp(self):
skip_if_no_xattrs()
_test_servers[0]._error_limiting = {} # clear out errors
_test_servers[0].error_limiter.stats.clear() # clear out errors
self.logger = debug_logger('proxy-ut')
self.app = proxy_server.Application(
None,
@ -3022,7 +3025,7 @@ class TestReplicatedObjectController(
bytes_before_timeout[0] = 700
kaboomed[0] = 0
sabotaged[0] = False
prosrv._error_limiting = {} # clear out errors
prosrv.error_limiter.stats.clear() # clear out errors
with mock.patch.object(proxy_base,
'http_response_to_document_iters',
sabotaged_hrtdi): # perma-broken
@ -3062,7 +3065,7 @@ class TestReplicatedObjectController(
bytes_before_timeout[0] = 300
kaboomed[0] = 0
sabotaged[0] = False
prosrv._error_limiting = {} # clear out errors
prosrv.error_limiter.stats.clear() # clear out errors
with mock.patch.object(proxy_base,
'http_response_to_document_iters',
single_sabotage_hrtdi):
@ -3102,7 +3105,7 @@ class TestReplicatedObjectController(
bytes_before_timeout[0] = 501
kaboomed[0] = 0
sabotaged[0] = False
prosrv._error_limiting = {} # clear out errors
prosrv.error_limiter.stats.clear() # clear out errors
with mock.patch.object(proxy_base,
'http_response_to_document_iters',
single_sabotage_hrtdi):
@ -3142,7 +3145,7 @@ class TestReplicatedObjectController(
bytes_before_timeout[0] = 750
kaboomed[0] = 0
sabotaged[0] = False
prosrv._error_limiting = {} # clear out errors
prosrv.error_limiter.stats.clear() # clear out errors
with mock.patch.object(proxy_base,
'http_response_to_document_iters',
single_sabotage_hrtdi):
@ -5081,7 +5084,7 @@ class TestReplicatedObjectController(
self.app.log_handoffs = True
self.app.logger.clear() # clean capture state
self.app.request_node_count = lambda r: 7
self.app._error_limiting = {} # clear out errors
self.app.error_limiter.stats.clear() # clear out errors
set_node_errors(self.app, object_ring._devs[0], 999,
last_error=(2 ** 63 - 1))
@ -5100,7 +5103,7 @@ class TestReplicatedObjectController(
self.app.log_handoffs = True
self.app.logger.clear() # clean capture state
self.app.request_node_count = lambda r: 7
self.app._error_limiting = {} # clear out errors
self.app.error_limiter.stats.clear() # clear out errors
for i in range(2):
set_node_errors(self.app, object_ring._devs[i], 999,
last_error=(2 ** 63 - 1))
@ -5125,7 +5128,7 @@ class TestReplicatedObjectController(
self.app.logger.clear() # clean capture state
self.app.request_node_count = lambda r: 10
object_ring.set_replicas(4) # otherwise we run out of handoffs
self.app._error_limiting = {} # clear out errors
self.app.error_limiter.stats.clear() # clear out errors
for i in range(4):
set_node_errors(self.app, object_ring._devs[i], 999,
last_error=(2 ** 63 - 1))
@ -5290,12 +5293,12 @@ class TestReplicatedObjectController(
self.assertTrue(
node_last_error(controller.app, object_ring.devs[0])
is not None)
for _junk in range(self.app.error_suppression_limit):
for _junk in range(self.app.error_limiter.suppression_limit):
self.assert_status_map(controller.HEAD, (200, 200, 503, 503,
503), 503)
self.assertEqual(
node_error_count(controller.app, object_ring.devs[0]),
self.app.error_suppression_limit + 1)
self.app.error_limiter.suppression_limit + 1)
self.assert_status_map(controller.HEAD, (200, 200, 200, 200, 200),
503)
self.assertTrue(
@ -5308,7 +5311,7 @@ class TestReplicatedObjectController(
202), 503)
self.assert_status_map(controller.DELETE,
(200, 200, 200, 204, 204, 204), 503)
self.app.error_suppression_interval = -300
self.app.error_limiter.suppression_interval = -300
self.assert_status_map(controller.HEAD, (200, 200, 200, 200, 200),
200)
self.assertRaises(BaseException,
@ -5329,12 +5332,12 @@ class TestReplicatedObjectController(
self.assertTrue(
node_last_error(controller.app, object_ring.devs[0])
is not None)
for _junk in range(self.app.error_suppression_limit):
for _junk in range(self.app.error_limiter.suppression_limit):
self.assert_status_map(controller.HEAD, (200, 200, 503, 503,
503), 503)
self.assertEqual(
node_error_count(controller.app, object_ring.devs[0]),
self.app.error_suppression_limit + 1)
self.app.error_limiter.suppression_limit + 1)
# wipe out any state in the ring
for policy in POLICIES:
@ -5360,10 +5363,9 @@ class TestReplicatedObjectController(
self.assertEqual(node_error_count(controller.app, odevs[0]), 2)
self.assertEqual(node_error_count(controller.app, odevs[1]), 0)
self.assertEqual(node_error_count(controller.app, odevs[2]), 0)
self.assertTrue(
node_last_error(controller.app, odevs[0]) is not None)
self.assertTrue(node_last_error(controller.app, odevs[1]) is None)
self.assertTrue(node_last_error(controller.app, odevs[2]) is None)
self.assertIsNotNone(node_last_error(controller.app, odevs[0]))
self.assertIsNone(node_last_error(controller.app, odevs[1]))
self.assertIsNone(node_last_error(controller.app, odevs[2]))
def test_PUT_error_limiting_last_node(self):
with save_globals():
@ -5380,14 +5382,13 @@ class TestReplicatedObjectController(
self.assertEqual(node_error_count(controller.app, odevs[0]), 0)
self.assertEqual(node_error_count(controller.app, odevs[1]), 0)
self.assertEqual(node_error_count(controller.app, odevs[2]), 2)
self.assertTrue(node_last_error(controller.app, odevs[0]) is None)
self.assertTrue(node_last_error(controller.app, odevs[1]) is None)
self.assertTrue(
node_last_error(controller.app, odevs[2]) is not None)
self.assertIsNone(node_last_error(controller.app, odevs[0]))
self.assertIsNone(node_last_error(controller.app, odevs[1]))
self.assertIsNotNone(node_last_error(controller.app, odevs[2]))
def test_acc_or_con_missing_returns_404(self):
with save_globals():
self.app._error_limiting = {}
self.app.error_limiter.stats.clear()
controller = ReplicatedObjectController(
self.app, 'account', 'container', 'object')
set_http_connect(200, 200, 200, 200, 200, 200)
@ -5455,7 +5456,8 @@ class TestReplicatedObjectController(
for dev in self.app.account_ring.devs:
set_node_errors(
self.app, dev, self.app.error_suppression_limit + 1,
self.app, dev,
self.app.error_limiter.suppression_limit + 1,
time.time())
set_http_connect(200)
# acct [isn't actually called since everything
@ -5469,9 +5471,10 @@ class TestReplicatedObjectController(
for dev in self.app.account_ring.devs:
set_node_errors(self.app, dev, 0, last_error=None)
for dev in self.app.container_ring.devs:
set_node_errors(self.app, dev,
self.app.error_suppression_limit + 1,
time.time())
set_node_errors(
self.app, dev,
self.app.error_limiter.suppression_limit + 1,
time.time())
set_http_connect(200, 200)
# acct cont [isn't actually called since
# everything is error limited]
@ -8039,7 +8042,7 @@ class TestECMismatchedFA(unittest.TestCase):
def tearDown(self):
prosrv = _test_servers[0]
# don't leak error limits and poison other tests
prosrv._error_limiting = {}
prosrv.error_limiter.stats.clear()
def test_mixing_different_objects_fragment_archives(self):
(prosrv, acc1srv, acc2srv, con1srv, con2srv, obj1srv,
@ -8077,7 +8080,7 @@ class TestECMismatchedFA(unittest.TestCase):
# Server obj1 will have the first version of the object (obj2 also
# gets it, but that gets stepped on later)
prosrv._error_limiting = {}
prosrv.error_limiter.stats.clear()
with mock.patch.object(obj3srv, 'PUT', bad_disk), \
mock.patch(
'swift.common.storage_policy.ECStoragePolicy.quorum'):
@ -8086,7 +8089,7 @@ class TestECMismatchedFA(unittest.TestCase):
self.assertEqual(resp.status_int, 201)
# Servers obj2 and obj3 will have the second version of the object.
prosrv._error_limiting = {}
prosrv.error_limiter.stats.clear()
with mock.patch.object(obj1srv, 'PUT', bad_disk), \
mock.patch(
'swift.common.storage_policy.ECStoragePolicy.quorum'):
@ -8098,7 +8101,7 @@ class TestECMismatchedFA(unittest.TestCase):
get_req = Request.blank("/v1/a/ec-crazytown/obj",
environ={"REQUEST_METHOD": "GET"},
headers={"X-Auth-Token": "t"})
prosrv._error_limiting = {}
prosrv.error_limiter.stats.clear()
with mock.patch.object(obj1srv, 'GET', bad_disk), \
mock.patch.object(obj2srv, 'GET', bad_disk):
resp = get_req.get_response(prosrv)
@ -8108,7 +8111,7 @@ class TestECMismatchedFA(unittest.TestCase):
get_req = Request.blank("/v1/a/ec-crazytown/obj",
environ={"REQUEST_METHOD": "GET"},
headers={"X-Auth-Token": "t"})
prosrv._error_limiting = {}
prosrv.error_limiter.stats.clear()
with mock.patch.object(obj1srv, 'GET', bad_disk):
resp = get_req.get_response(prosrv)
self.assertEqual(resp.status_int, 200)
@ -8118,7 +8121,7 @@ class TestECMismatchedFA(unittest.TestCase):
get_req = Request.blank("/v1/a/ec-crazytown/obj",
environ={"REQUEST_METHOD": "GET"},
headers={"X-Auth-Token": "t"})
prosrv._error_limiting = {}
prosrv.error_limiter.stats.clear()
with mock.patch.object(obj2srv, 'GET', bad_disk):
resp = get_req.get_response(prosrv)
self.assertEqual(resp.status_int, 503)
@ -8159,7 +8162,7 @@ class TestECMismatchedFA(unittest.TestCase):
# First subset of object server will have the first version of the
# object
prosrv._error_limiting = {}
prosrv.error_limiter.stats.clear()
with mock.patch.object(obj4srv, 'PUT', bad_disk), \
mock.patch.object(obj5srv, 'PUT', bad_disk), \
mock.patch.object(obj6srv, 'PUT', bad_disk), \
@ -8170,7 +8173,7 @@ class TestECMismatchedFA(unittest.TestCase):
self.assertEqual(resp.status_int, 201)
# Second subset will have the second version of the object.
prosrv._error_limiting = {}
prosrv.error_limiter.stats.clear()
with mock.patch.object(obj1srv, 'PUT', bad_disk), \
mock.patch.object(obj2srv, 'PUT', bad_disk), \
mock.patch.object(obj3srv, 'PUT', bad_disk), \
@ -8184,7 +8187,7 @@ class TestECMismatchedFA(unittest.TestCase):
get_req = Request.blank("/v1/a/ec-dup-crazytown/obj",
environ={"REQUEST_METHOD": "GET"},
headers={"X-Auth-Token": "t"})
prosrv._error_limiting = {}
prosrv.error_limiter.stats.clear()
with mock.patch.object(obj2srv, 'GET', bad_disk), \
mock.patch.object(obj3srv, 'GET', bad_disk), \
mock.patch.object(obj4srv, 'GET', bad_disk), \
@ -8197,7 +8200,7 @@ class TestECMismatchedFA(unittest.TestCase):
get_req = Request.blank("/v1/a/ec-dup-crazytown/obj",
environ={"REQUEST_METHOD": "GET"},
headers={"X-Auth-Token": "t"})
prosrv._error_limiting = {}
prosrv.error_limiter.stats.clear()
with mock.patch.object(obj1srv, 'GET', bad_disk), \
mock.patch.object(obj2srv, 'GET', bad_disk), \
mock.patch.object(obj3srv, 'GET', bad_disk), \
@ -8210,7 +8213,7 @@ class TestECMismatchedFA(unittest.TestCase):
get_req = Request.blank("/v1/a/ec-dup-crazytown/obj",
environ={"REQUEST_METHOD": "GET"},
headers={"X-Auth-Token": "t"})
prosrv._error_limiting = {}
prosrv.error_limiter.stats.clear()
with mock.patch.object(obj2srv, 'GET', bad_disk), \
mock.patch.object(obj3srv, 'GET', bad_disk), \
mock.patch.object(obj4srv, 'GET', bad_disk), \
@ -8229,7 +8232,7 @@ class TestECGets(unittest.TestCase):
rmtree(self.tempdir, ignore_errors=True)
prosrv = _test_servers[0]
# don't leak error limits and poison other tests
prosrv._error_limiting = {}
prosrv.error_limiter.stats.clear()
super(TestECGets, self).tearDown()
def _setup_nodes_and_do_GET(self, objs, node_state):
@ -8542,12 +8545,12 @@ class TestObjectDisconnectCleanup(unittest.TestCase):
skip_if_no_xattrs()
debug.hub_exceptions(False)
self._cleanup_devices()
_test_servers[0]._error_limiting = {} # clear out errors
_test_servers[0].error_limiter.stats.clear() # clear out errors
def tearDown(self):
debug.hub_exceptions(True)
self._cleanup_devices()
_test_servers[0]._error_limiting = {} # clear out errors
_test_servers[0].error_limiter.stats.clear() # clear out errors
def _check_disconnect_cleans_up(self, policy_name, is_chunked=False):
proxy_port = _test_sockets[0].getsockname()[1]
@ -8640,7 +8643,7 @@ class TestObjectDisconnectCleanup(unittest.TestCase):
class TestObjectECRangedGET(unittest.TestCase):
def setUp(self):
_test_servers[0].logger._clear()
_test_servers[0]._error_limiting = {} # clear out errors
_test_servers[0].error_limiter.stats.clear() # clear out errors
self.app = proxy_server.Application(
None,
logger=debug_logger('proxy-ut'),
@ -8651,7 +8654,7 @@ class TestObjectECRangedGET(unittest.TestCase):
prosrv = _test_servers[0]
self.assertFalse(prosrv.logger.get_lines_for_level('error'))
self.assertFalse(prosrv.logger.get_lines_for_level('warning'))
prosrv._error_limiting = {} # clear out errors
prosrv.error_limiter.stats.clear() # clear out errors
@classmethod
def setUpClass(cls):
@ -9672,7 +9675,7 @@ class TestContainerController(unittest.TestCase):
def test_acc_missing_returns_404(self):
for meth in ('DELETE', 'PUT'):
with save_globals():
self.app._error_limiting = {}
self.app.error_limiter.stats.clear()
controller = proxy_server.ContainerController(self.app,
'account',
'container')
@ -9709,9 +9712,10 @@ class TestContainerController(unittest.TestCase):
self.assertEqual(resp.status_int, 404)
for dev in self.app.account_ring.devs:
set_node_errors(self.app, dev,
self.app.error_suppression_limit + 1,
time.time())
set_node_errors(
self.app, dev,
self.app.error_limiter.suppression_limit + 1,
time.time())
set_http_connect(200, 200, 200, 200, 200, 200)
# Make sure it is a blank request wthout env caching
req = Request.blank('/v1/a/c',
@ -9759,12 +9763,12 @@ class TestContainerController(unittest.TestCase):
self.assertTrue(
node_last_error(controller.app, container_ring.devs[0])
is not None)
for _junk in range(self.app.error_suppression_limit):
for _junk in range(self.app.error_limiter.suppression_limit):
self.assert_status_map(controller.HEAD,
(200, 503, 503, 503), 503)
self.assertEqual(
node_error_count(controller.app, container_ring.devs[0]),
self.app.error_suppression_limit + 1)
self.app.error_limiter.suppression_limit + 1)
self.assert_status_map(controller.HEAD, (200, 200, 200, 200), 503)
self.assertTrue(
node_last_error(controller.app, container_ring.devs[0])
@ -9773,7 +9777,7 @@ class TestContainerController(unittest.TestCase):
missing_container=True)
self.assert_status_map(controller.DELETE,
(200, 204, 204, 204), 503)
self.app.error_suppression_interval = -300
self.app.error_limiter.suppression_interval = -300
self.assert_status_map(controller.HEAD, (200, 200, 200, 200), 200)
self.assert_status_map(controller.DELETE, (200, 204, 204, 204),
404, raise_exc=True)
@ -11324,7 +11328,7 @@ class TestProxyObjectPerformance(unittest.TestCase):
# various data paths between the proxy server and the object
# server. Used as a play ground to debug buffer sizes for sockets.
skip_if_no_xattrs()
_test_servers[0]._error_limiting = {} # clear out errors
_test_servers[0].error_limiter.stats.clear() # clear out errors
prolis = _test_sockets[0]
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
# Client is transmitting in 2 MB chunks
@ -11445,7 +11449,7 @@ class TestSocketObjectVersions(unittest.TestCase):
def setUp(self):
global _test_sockets
skip_if_no_xattrs()
_test_servers[0]._error_limiting = {} # clear out errors
_test_servers[0].error_limiter.stats.clear() # clear out errors
self.prolis = prolis = listen_zero()
self._orig_prolis = _test_sockets[0]
allowed_headers = ', '.join([
@ -11480,7 +11484,7 @@ class TestSocketObjectVersions(unittest.TestCase):
global _test_sockets
self.sockets[0] = self._orig_prolis
_test_sockets = tuple(self.sockets)
_test_servers[0]._error_limiting = {} # clear out errors
_test_servers[0].error_limiter.stats.clear() # clear out errors
def test_version_manifest(self, oc=b'versions', vc=b'vers', o=b'name'):
versions_to_create = 3