diff --git a/doc/source/ratelimit.rst b/doc/source/ratelimit.rst index 6d962a1735..aa318d8008 100644 --- a/doc/source/ratelimit.rst +++ b/doc/source/ratelimit.rst @@ -81,3 +81,20 @@ Container Size Rate Limit ================ ============ +----------------------------- +Account Specific Ratelimiting +----------------------------- + +The above ratelimiting is to prevent the "many writes to a single container" +bottleneck from causing a problem. There could also be a problem where a single +account is just using too much of the cluster's resources. In this case, the +container ratelimits may not help because the customer could be doing thousands +of reqs/sec to distributed containers each getting a small fraction of the +total so those limits would never trigger. If a system adminstrator notices +this, he/she can set the X-Account-Sysmeta-Global-Write-Ratelimit on an account +and that will limit the total number of write requests (PUT, POST, DELETE, +COPY) that account can do for the whole account. This limit will be in addition +to the applicable account/container limits from above. This header will be +hidden from the user, because of the gatekeeper middleware, and can only be set +using a direct client to the account nodes. It accepts a float value and will +only limit requests if the value is > 0. diff --git a/swift/common/middleware/ratelimit.py b/swift/common/middleware/ratelimit.py index 2f8a6b5d71..b88fbccc88 100644 --- a/swift/common/middleware/ratelimit.py +++ b/swift/common/middleware/ratelimit.py @@ -18,7 +18,8 @@ from swift import gettext_ as _ import eventlet from swift.common.utils import cache_from_env, get_logger, register_swift_info -from swift.proxy.controllers.base import get_container_memcache_key +from swift.proxy.controllers.base import get_container_memcache_key, \ + get_account_info from swift.common.memcached import MemcacheConnectionError from swift.common.swob import Request, Response @@ -117,13 +118,13 @@ class RateLimitMiddleware(object): 'object_count', container_info.get('container_size', 0)) return rv - def get_ratelimitable_key_tuples(self, req_method, account_name, + def get_ratelimitable_key_tuples(self, req, account_name, container_name=None, obj_name=None): """ Returns a list of key (used in memcache), ratelimit tuples. Keys should be checked in order. - :param req_method: HTTP method + :param req: swob request :param account_name: account name from path :param container_name: container name from path :param obj_name: object name from path @@ -132,12 +133,12 @@ class RateLimitMiddleware(object): # COPYs are not limited if self.account_ratelimit and \ account_name and container_name and not obj_name and \ - req_method in ('PUT', 'DELETE'): + req.method in ('PUT', 'DELETE'): keys.append(("ratelimit/%s" % account_name, self.account_ratelimit)) if account_name and container_name and obj_name and \ - req_method in ('PUT', 'DELETE', 'POST'): + req.method in ('PUT', 'DELETE', 'POST', 'COPY'): container_size = self.get_container_size( account_name, container_name) container_rate = get_maxrate( @@ -148,7 +149,7 @@ class RateLimitMiddleware(object): container_rate)) if account_name and container_name and not obj_name and \ - req_method == 'GET': + req.method == 'GET': container_size = self.get_container_size( account_name, container_name) container_rate = get_maxrate( @@ -158,6 +159,20 @@ class RateLimitMiddleware(object): "ratelimit_listing/%s/%s" % (account_name, container_name), container_rate)) + if account_name and req.method in ('PUT', 'DELETE', 'POST', 'COPY'): + account_info = get_account_info(req.environ, self.app) + account_global_ratelimit = \ + account_info.get('sysmeta', {}).get('global-write-ratelimit') + if account_global_ratelimit: + try: + account_global_ratelimit = float(account_global_ratelimit) + if account_global_ratelimit > 0: + keys.append(( + "ratelimit/global-write/%s" % account_name, + account_global_ratelimit)) + except ValueError: + pass + return keys def _get_sleep_time(self, key, max_rate): @@ -218,7 +233,7 @@ class RateLimitMiddleware(object): if account_name in self.ratelimit_whitelist: return None for key, max_rate in self.get_ratelimitable_key_tuples( - req.method, account_name, container_name=container_name, + req, account_name, container_name=container_name, obj_name=obj_name): try: need_to_sleep = self._get_sleep_time(key, max_rate) diff --git a/test/unit/common/middleware/test_ratelimit.py b/test/unit/common/middleware/test_ratelimit.py index d80895834a..25b1258000 100644 --- a/test/unit/common/middleware/test_ratelimit.py +++ b/test/unit/common/middleware/test_ratelimit.py @@ -16,6 +16,7 @@ import unittest import time import eventlet +import mock from contextlib import contextmanager from threading import Thread @@ -194,16 +195,45 @@ class TestRateLimit(unittest.TestCase): the_app = ratelimit.RateLimitMiddleware(None, conf_dict, logger=FakeLogger()) the_app.memcache_client = fake_memcache - self.assertEquals(len(the_app.get_ratelimitable_key_tuples( - 'DELETE', 'a', None, None)), 0) - self.assertEquals(len(the_app.get_ratelimitable_key_tuples( - 'PUT', 'a', 'c', None)), 1) - self.assertEquals(len(the_app.get_ratelimitable_key_tuples( - 'DELETE', 'a', 'c', None)), 1) - self.assertEquals(len(the_app.get_ratelimitable_key_tuples( - 'GET', 'a', 'c', 'o')), 0) - self.assertEquals(len(the_app.get_ratelimitable_key_tuples( - 'PUT', 'a', 'c', 'o')), 1) + req = lambda: None + req.environ = {} + with mock.patch('swift.common.middleware.ratelimit.get_account_info', + lambda *args, **kwargs: {}): + req.method = 'DELETE' + self.assertEquals(len(the_app.get_ratelimitable_key_tuples( + req, 'a', None, None)), 0) + req.method = 'PUT' + self.assertEquals(len(the_app.get_ratelimitable_key_tuples( + req, 'a', 'c', None)), 1) + req.method = 'DELETE' + self.assertEquals(len(the_app.get_ratelimitable_key_tuples( + req, 'a', 'c', None)), 1) + req.method = 'GET' + self.assertEquals(len(the_app.get_ratelimitable_key_tuples( + req, 'a', 'c', 'o')), 0) + req.method = 'PUT' + self.assertEquals(len(the_app.get_ratelimitable_key_tuples( + req, 'a', 'c', 'o')), 1) + + def get_fake_ratelimit(*args, **kwargs): + return {'sysmeta': {'global-write-ratelimit': 10}} + + with mock.patch('swift.common.middleware.ratelimit.get_account_info', + get_fake_ratelimit): + req.method = 'PUT' + self.assertEquals(len(the_app.get_ratelimitable_key_tuples( + req, 'a', 'c', None)), 2) + self.assertEquals(the_app.get_ratelimitable_key_tuples( + req, 'a', 'c', None)[1], ('ratelimit/global-write/a', 10)) + + def get_fake_ratelimit(*args, **kwargs): + return {'sysmeta': {'global-write-ratelimit': 'notafloat'}} + + with mock.patch('swift.common.middleware.ratelimit.get_account_info', + get_fake_ratelimit): + req.method = 'PUT' + self.assertEquals(len(the_app.get_ratelimitable_key_tuples( + req, 'a', 'c', None)), 1) def test_memcached_container_info_dict(self): mdict = headers_to_container_info({'x-container-object-count': '45'}) @@ -219,8 +249,13 @@ class TestRateLimit(unittest.TestCase): the_app = ratelimit.RateLimitMiddleware(None, conf_dict, logger=FakeLogger()) the_app.memcache_client = fake_memcache - tuples = the_app.get_ratelimitable_key_tuples('PUT', 'a', 'c', 'o') - self.assertEquals(tuples, [('ratelimit/a/c', 200.0)]) + req = lambda: None + req.method = 'PUT' + req.environ = {} + with mock.patch('swift.common.middleware.ratelimit.get_account_info', + lambda *args, **kwargs: {}): + tuples = the_app.get_ratelimitable_key_tuples(req, 'a', 'c', 'o') + self.assertEquals(tuples, [('ratelimit/a/c', 200.0)]) def test_account_ratelimit(self): current_rate = 5 @@ -228,18 +263,20 @@ class TestRateLimit(unittest.TestCase): conf_dict = {'account_ratelimit': current_rate} self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) - for meth, exp_time in [ - ('DELETE', 9.8), ('GET', 0), ('POST', 0), ('PUT', 9.8)]: - req = Request.blank('/v/a%s/c' % meth) - req.method = meth - req.environ['swift.cache'] = FakeMemcache() - make_app_call = lambda: self.test_ratelimit(req.environ, - start_response) - begin = time.time() - self._run(make_app_call, num_calls, current_rate, - check_time=bool(exp_time)) - self.assertEquals(round(time.time() - begin, 1), exp_time) - self._reset_time() + with mock.patch('swift.common.middleware.ratelimit.get_account_info', + lambda *args, **kwargs: {}): + for meth, exp_time in [ + ('DELETE', 9.8), ('GET', 0), ('POST', 0), ('PUT', 9.8)]: + req = Request.blank('/v/a%s/c' % meth) + req.method = meth + req.environ['swift.cache'] = FakeMemcache() + make_app_call = lambda: self.test_ratelimit(req.environ, + start_response) + begin = time.time() + self._run(make_app_call, num_calls, current_rate, + check_time=bool(exp_time)) + self.assertEquals(round(time.time() - begin, 1), exp_time) + self._reset_time() def test_ratelimit_set_incr(self): current_rate = 5 @@ -254,8 +291,10 @@ class TestRateLimit(unittest.TestCase): make_app_call = lambda: self.test_ratelimit(req.environ, start_response) begin = time.time() - self._run(make_app_call, num_calls, current_rate, check_time=False) - self.assertEquals(round(time.time() - begin, 1), 9.8) + with mock.patch('swift.common.middleware.ratelimit.get_account_info', + lambda *args, **kwargs: {}): + self._run(make_app_call, num_calls, current_rate, check_time=False) + self.assertEquals(round(time.time() - begin, 1), 9.8) def test_ratelimit_whitelist(self): global time_ticker @@ -342,18 +381,20 @@ class TestRateLimit(unittest.TestCase): time_override = [0, 0, 0, 0, None] # simulates 4 requests coming in at same time, then sleeping - r = self.test_ratelimit(req.environ, start_response) - mock_sleep(.1) - r = self.test_ratelimit(req.environ, start_response) - mock_sleep(.1) - r = self.test_ratelimit(req.environ, start_response) - self.assertEquals(r[0], 'Slow down') - mock_sleep(.1) - r = self.test_ratelimit(req.environ, start_response) - self.assertEquals(r[0], 'Slow down') - mock_sleep(.1) - r = self.test_ratelimit(req.environ, start_response) - self.assertEquals(r[0], '204 No Content') + with mock.patch('swift.common.middleware.ratelimit.get_account_info', + lambda *args, **kwargs: {}): + r = self.test_ratelimit(req.environ, start_response) + mock_sleep(.1) + r = self.test_ratelimit(req.environ, start_response) + mock_sleep(.1) + r = self.test_ratelimit(req.environ, start_response) + self.assertEquals(r[0], 'Slow down') + mock_sleep(.1) + r = self.test_ratelimit(req.environ, start_response) + self.assertEquals(r[0], 'Slow down') + mock_sleep(.1) + r = self.test_ratelimit(req.environ, start_response) + self.assertEquals(r[0], '204 No Content') def test_ratelimit_max_rate_double_container(self): global time_ticker @@ -374,18 +415,20 @@ class TestRateLimit(unittest.TestCase): time_override = [0, 0, 0, 0, None] # simulates 4 requests coming in at same time, then sleeping - r = self.test_ratelimit(req.environ, start_response) - mock_sleep(.1) - r = self.test_ratelimit(req.environ, start_response) - mock_sleep(.1) - r = self.test_ratelimit(req.environ, start_response) - self.assertEquals(r[0], 'Slow down') - mock_sleep(.1) - r = self.test_ratelimit(req.environ, start_response) - self.assertEquals(r[0], 'Slow down') - mock_sleep(.1) - r = self.test_ratelimit(req.environ, start_response) - self.assertEquals(r[0], '204 No Content') + with mock.patch('swift.common.middleware.ratelimit.get_account_info', + lambda *args, **kwargs: {}): + r = self.test_ratelimit(req.environ, start_response) + mock_sleep(.1) + r = self.test_ratelimit(req.environ, start_response) + mock_sleep(.1) + r = self.test_ratelimit(req.environ, start_response) + self.assertEquals(r[0], 'Slow down') + mock_sleep(.1) + r = self.test_ratelimit(req.environ, start_response) + self.assertEquals(r[0], 'Slow down') + mock_sleep(.1) + r = self.test_ratelimit(req.environ, start_response) + self.assertEquals(r[0], '204 No Content') def test_ratelimit_max_rate_double_container_listing(self): global time_ticker @@ -431,6 +474,7 @@ class TestRateLimit(unittest.TestCase): the_app.memcache_client = fake_memcache req = lambda: None req.method = 'PUT' + req.environ = {} class rate_caller(Thread): @@ -443,18 +487,20 @@ class TestRateLimit(unittest.TestCase): self.result = the_app.handle_ratelimit(req, self.myname, 'c', None) - nt = 15 - begin = time.time() - threads = [] - for i in range(nt): - rc = rate_caller('a%s' % i) - rc.start() - threads.append(rc) - for thread in threads: - thread.join() + with mock.patch('swift.common.middleware.ratelimit.get_account_info', + lambda *args, **kwargs: {}): + nt = 15 + begin = time.time() + threads = [] + for i in range(nt): + rc = rate_caller('a%s' % i) + rc.start() + threads.append(rc) + for thread in threads: + thread.join() - time_took = time.time() - begin - self.assertEquals(1.5, round(time_took, 1)) + time_took = time.time() - begin + self.assertEquals(1.5, round(time_took, 1)) def test_call_invalid_path(self): env = {'REQUEST_METHOD': 'GET', @@ -504,9 +550,11 @@ class TestRateLimit(unittest.TestCase): make_app_call = lambda: self.test_ratelimit(req.environ, start_response) begin = time.time() - self._run(make_app_call, num_calls, current_rate, check_time=False) - time_took = time.time() - begin - self.assertEquals(round(time_took, 1), 0) # no memcache, no limiting + with mock.patch('swift.common.middleware.ratelimit.get_account_info', + lambda *args, **kwargs: {}): + self._run(make_app_call, num_calls, current_rate, check_time=False) + time_took = time.time() - begin + self.assertEquals(round(time_took, 1), 0) # no memcache, no limit if __name__ == '__main__': unittest.main()