diff --git a/doc/source/deployment_guide.rst b/doc/source/deployment_guide.rst index df5b4f642d..eab0432ae6 100644 --- a/doc/source/deployment_guide.rst +++ b/doc/source/deployment_guide.rst @@ -470,16 +470,6 @@ error_suppression_interval 60 Time in seconds that must no longer error limited error_suppression_limit 10 Error count to consider a node error limited -rate_limit 20000.0 Max container level ops per - second -account_rate_limit 200.0 Max account level ops per - second -rate_limit_account_whitelist Comma separated list of - account name hashes to not - rate limit -rate_limit_account_blacklist Comma separated list of - account name hashes to block - completely ============================ =============== ============================= [auth] diff --git a/doc/source/index.rst b/doc/source/index.rst index 6e5a7f6592..66f4d1cc7a 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -25,7 +25,7 @@ Overview: overview_auth overview_replication overview_stats - rate_limiting + ratelimit Development: diff --git a/doc/source/rate_limiting.rst b/doc/source/rate_limiting.rst deleted file mode 100644 index 700b9cd2fd..0000000000 --- a/doc/source/rate_limiting.rst +++ /dev/null @@ -1,67 +0,0 @@ -============= -Rate Limiting -============= - -Rate limiting in swift is implemented as a pluggable middleware. Rate -limiting is performed on requests that result in database writes to the -account and container sqlite dbs. It uses memcached and is dependant on -the proxy servers having highly synchronized time. The rate limits are -limited by the accuracy of the proxy server clocks. - --------------- -Configuration --------------- - -All configuration is optional. If no account or container limits are provided -there will be no rate limiting. Configuration available: - -====================== ========= ============================================= -Option Default Description ----------------------- --------- --------------------------------------------- -clock_accuracy 1000 Represents how accurate the proxy servers' - system clocks are with each other. 1000 means - that all the proxies' clock are accurate to - each other within 1 millisecond. No - ratelimit should be higher than the clock - accuracy. -max_sleep_time_seconds 60 App will immediately return a 498 response - if the necessary sleep time ever exceeds - the given max_sleep_time_seconds. -account_ratelimit 0 If set, will limit all requests to - /account_name and PUTs to - /account_name/container_name. Number is in - requests per second -account_whitelist '' Comma separated lists of account names that - will not be rate limited. -account_blacklist '' Comma separated lists of account names that - will not be allowed. Returns a 497 response. -container_limit_size '' When set with container_limit_x = r: - for containers of size x, limit requests per - second to r. Will limit GET and HEAD - requests to /account_name/container_name and - PUTs and DELETEs to - /account_name/container_name/object_name -====================== ========= ============================================= - -The container rate limits are linearly interpolated from the values given. A -sample container rate limiting could be: - -container_limit_100 = 100 - -container_limit_200 = 50 - -container_limit_500 = 20 - -This would result in - -================ ============ -Container Size Rate Limit ----------------- ------------ -0-99 No limiting -100 100 -150 75 -500 20 -1000 20 -================ ============ - - diff --git a/doc/source/ratelimit.rst b/doc/source/ratelimit.rst new file mode 100644 index 0000000000..43649e55e5 --- /dev/null +++ b/doc/source/ratelimit.rst @@ -0,0 +1,67 @@ +============= +Rate Limiting +============= + +Rate limiting in swift is implemented as a pluggable middleware. Rate +limiting is performed on requests that result in database writes to the +account and container sqlite dbs. It uses memcached and is dependant on +the proxy servers having highly synchronized time. The rate limits are +limited by the accuracy of the proxy server clocks. + +-------------- +Configuration +-------------- + +All configuration is optional. If no account or container limits are provided +there will be no rate limiting. Configuration available: + +======================== ========= =========================================== +Option Default Description +---------------------- --------- ------------------------------------------- +clock_accuracy 1000 Represents how accurate the proxy servers' + system clocks are with each other. 1000 + means that all the proxies' clock are + accurate to each other within 1 + millisecond. No ratelimit should be + higher than the clock accuracy. +max_sleep_time_seconds 60 App will immediately return a 498 response + if the necessary sleep time ever exceeds + the given max_sleep_time_seconds. +account_ratelimit 0 If set, will limit all requests to + /account_name and PUTs to + /account_name/container_name. Number is in + requests per second +account_whitelist '' Comma separated lists of account names that + will not be rate limited. +account_blacklist '' Comma separated lists of account names that + will not be allowed. Returns a 497 response. +container_ratelimit_size '' When set with container_limit_x = r: + for containers of size x, limit requests + per second to r. Will limit GET and HEAD + requests to /account_name/container_name and + PUTs and DELETEs to + /account_name/container_name/object_name +======================== ========= =========================================== + +The container rate limits are linearly interpolated from the values given. A +sample container rate limiting could be: + +container_ratelimit_100 = 100 + +container_ratelimit_200 = 50 + +container_ratelimit_500 = 20 + +This would result in + +================ ============ +Container Size Rate Limit +---------------- ------------ +0-99 No limiting +100 100 +150 75 +500 20 +1000 20 +================ ============ + + diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 538d91933e..21a24ecaaa 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -71,6 +71,6 @@ use = egg:swift#ratelimit # for containers of size x limit requests per second to r. The container # rate will be linearly interpolated from the values given. With the values # below, a container of size 5 will get a rate of 75. -# container_limit_0 = 100 -# container_limit_10 = 50 -# container_limit_50 = 20 +# container_ratelimit_0 = 100 +# container_ratelimit_10 = 50 +# container_ratelimit_50 = 20 diff --git a/swift/common/middleware/ratelimit.py b/swift/common/middleware/ratelimit.py index 3fb90641ef..ca0cd6e427 100644 --- a/swift/common/middleware/ratelimit.py +++ b/swift/common/middleware/ratelimit.py @@ -37,26 +37,26 @@ class RateLimitMiddleware(object): self.logger = logger else: self.logger = get_logger(conf) - self.account_rate_limit = float(conf.get('account_ratelimit', 0)) + self.account_ratelimit = float(conf.get('account_ratelimit', 0)) self.max_sleep_time_seconds = float(conf.get('max_sleep_time_seconds', 60)) self.clock_accuracy = int(conf.get('clock_accuracy', 1000)) - self.rate_limit_whitelist = [acc.strip() for acc in + self.ratelimit_whitelist = [acc.strip() for acc in conf.get('account_whitelist', '').split(',') if acc.strip()] - self.rate_limit_blacklist = [acc.strip() for acc in + self.ratelimit_blacklist = [acc.strip() for acc in conf.get('account_blacklist', '').split(',') if acc.strip()] self.memcache_client = None conf_limits = [] for conf_key in conf.keys(): - if conf_key.startswith('container_limit_'): - cont_size = int(conf_key[len('container_limit_'):]) + if conf_key.startswith('container_ratelimit_'): + cont_size = int(conf_key[len('container_ratelimit_'):]) rate = float(conf[conf_key]) conf_limits.append((cont_size, rate)) conf_limits.sort() - self.container_limits = [] + self.container_ratelimits = [] while conf_limits: cur_size, cur_rate = conf_limits.pop(0) if conf_limits: @@ -71,7 +71,7 @@ class RateLimitMiddleware(object): else: line_func = lambda x: cur_rate - self.container_limits.append((cur_size, cur_rate, line_func)) + self.container_ratelimits.append((cur_size, cur_rate, line_func)) def get_container_maxrate(self, container_size): """ @@ -80,11 +80,10 @@ class RateLimitMiddleware(object): last_func = None if container_size: container_size = int(container_size) - for size, rate, func in self.container_limits: + for size, rate, func in self.container_ratelimits: if container_size < size: break last_func = func - if last_func: return last_func(container_size) return None @@ -102,11 +101,11 @@ class RateLimitMiddleware(object): :param obj_name: object name from path """ keys = [] - if self.account_rate_limit and account_name and ( + if self.account_ratelimit and account_name and ( not (container_name or obj_name) or (container_name and not obj_name and req_method == 'PUT')): keys.append(("ratelimit/%s" % account_name, - self.account_rate_limit)) + self.account_ratelimit)) if account_name and container_name and ( (not obj_name and req_method in ('GET', 'HEAD')) or @@ -155,7 +154,7 @@ class RateLimitMiddleware(object): return float(need_to_sleep_m) / self.clock_accuracy - def handle_rate_limit(self, req, account_name, container_name, obj_name): + def handle_ratelimit(self, req, account_name, container_name, obj_name): ''' Performs rate limiting and account white/black listing. Sleeps if necessary. @@ -164,13 +163,12 @@ class RateLimitMiddleware(object): :param container_name: container name from path :param obj_name: object name from path ''' - if account_name in self.rate_limit_blacklist: + if account_name in self.ratelimit_blacklist: self.logger.error('Returning 497 because of blacklisting') return Response(status='497 Blacklisted', body='Your account has been blacklisted', request=req) - if account_name in self.rate_limit_whitelist: + if account_name in self.ratelimit_whitelist: return None - for key, max_rate in self.get_ratelimitable_key_tuples( req.method, account_name, @@ -186,7 +184,6 @@ class RateLimitMiddleware(object): error_resp = Response(status='498 Rate Limited', body='Slow down', request=req) return error_resp - return None def __call__(self, env, start_response): @@ -201,13 +198,11 @@ class RateLimitMiddleware(object): if self.memcache_client is None: self.memcache_client = cache_from_env(env) version, account, container, obj = split_path(req.path, 1, 4, True) - - rate_limit_resp = self.handle_rate_limit(req, account, container, - obj) - if rate_limit_resp is None: + ratelimit_resp = self.handle_ratelimit(req, account, container, obj) + if ratelimit_resp is None: return self.app(env, start_response) else: - return rate_limit_resp(env, start_response) + return ratelimit_resp(env, start_response) def filter_factory(global_conf, **local_conf): diff --git a/swift/proxy/server.py b/swift/proxy/server.py index b25e7e7008..d17c0659b2 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -871,7 +871,7 @@ class ContainerController(Controller): resp = self.GETorHEAD_base(req, 'Container', part, nodes, req.path_info, self.app.container_ring.replica_count) - # set the memcache container size for ratelimiting if missing + # set the memcache container size for ratelimiting cache_key = get_container_memcache_key(self.account_name, self.container_name) self.app.memcache.set(cache_key, diff --git a/test/unit/common/middleware/test_ratelimit.py b/test/unit/common/middleware/test_ratelimit.py index 2892b6cafc..7b8c91d73c 100644 --- a/test/unit/common/middleware/test_ratelimit.py +++ b/test/unit/common/middleware/test_ratelimit.py @@ -126,9 +126,9 @@ class TestRateLimit(unittest.TestCase): return time_diff def test_get_container_maxrate(self): - conf_dict = {'container_limit_10': 200, - 'container_limit_50': 100, - 'container_limit_75': 30} + conf_dict = {'container_ratelimit_10': 200, + 'container_ratelimit_50': 100, + 'container_ratelimit_75': 30} test_ratelimit = dummy_filter_factory(conf_dict)(FakeApp()) self.assertEquals(test_ratelimit.get_container_maxrate(0), None) self.assertEquals(test_ratelimit.get_container_maxrate(5), None) @@ -139,7 +139,7 @@ class TestRateLimit(unittest.TestCase): def test_get_ratelimitable_key_tuples(self): current_rate = 13 conf_dict = {'account_ratelimit': current_rate, - 'container_limit_3': 200} + 'container_ratelimit_3': 200} fake_memcache = FakeMemcache() fake_memcache.store[get_container_memcache_key('a', 'c')] = \ {'container_size': 5} @@ -303,8 +303,8 @@ class TestRateLimit(unittest.TestCase): def run(self): for j in range(num_calls): - self.result = the_app.handle_rate_limit(req, self.myname, - None, None) + self.result = the_app.handle_ratelimit(req, self.myname, + None, None) nt = 15 begin = time.time() @@ -323,9 +323,9 @@ class TestRateLimit(unittest.TestCase): conf_dict = {'clock_accuracy': 1000, 'account_ratelimit': 10, 'max_sleep_time_seconds': 4, - 'container_limit_10': 6, - 'container_limit_50': 2, - 'container_limit_75': 1} + 'container_ratelimit_10': 6, + 'container_ratelimit_50': 2, + 'container_ratelimit_75': 1} self.test_ratelimit = dummy_filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) req = Request.blank('/v/a/c')