openstack/swift
Code Issues Proposed changes

use get_container_info in ratelimit

Browse Source 
Everyone else cooperates to store the container_info cache in the env the
first time it's requested.  This should save a duplicate memcache hit on
requests that ratelimiting looks at.

Change-Id: Ic6411d4619db6b53fafe9fdbf1d0a370d1258c38
This commit is contained in:
Michael Barton 2014-09-25 16:29:57 +00:00
parent 813cc0fe0d
commit 781a162004
2 changed files with 26 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
swift/common/middleware/ratelimit.py
View File

@ -18,8 +18,7 @@ from swift import gettext_ as _
import eventlet import eventlet
from swift.common.utils import cache_from_env, get_logger, register_swift_info from swift.common.utils import cache_from_env, get_logger, register_swift_info
from swift.proxy.controllers.base import get_container_memcache_key, \ from swift.proxy.controllers.base import get_account_info, get_container_info
get_account_info
from swift.common.memcached import MemcacheConnectionError from swift.common.memcached import MemcacheConnectionError
from swift.common.swob import Request, Response from swift.common.swob import Request, Response
@ -118,11 +117,10 @@ class RateLimitMiddleware(object):
self.container_listing_ratelimits = interpret_conf_limits( self.container_listing_ratelimits = interpret_conf_limits(
conf, 'container_listing_ratelimit_') conf, 'container_listing_ratelimit_')
def get_container_size(self, account_name, container_name): def get_container_size(self, env):
rv = 0 rv = 0
memcache_key = get_container_memcache_key(account_name, container_info = get_container_info(
container_name) env, self.app, swift_source='RL')
container_info = self.memcache_client.get(memcache_key)
if isinstance(container_info, dict): if isinstance(container_info, dict):
rv = container_info.get( rv = container_info.get(
'object_count', container_info.get('container_size', 0)) 'object_count', container_info.get('container_size', 0))
@ -149,8 +147,7 @@ class RateLimitMiddleware(object):
if account_name and container_name and obj_name and \ if account_name and container_name and obj_name and \
req.method in ('PUT', 'DELETE', 'POST', 'COPY'): req.method in ('PUT', 'DELETE', 'POST', 'COPY'):
container_size = self.get_container_size( container_size = self.get_container_size(req.environ)
account_name, container_name)
container_rate = get_maxrate( container_rate = get_maxrate(
self.container_ratelimits, container_size) self.container_ratelimits, container_size)
if container_rate: if container_rate:
@ -160,8 +157,7 @@ class RateLimitMiddleware(object):
if account_name and container_name and not obj_name and \ if account_name and container_name and not obj_name and \
req.method == 'GET': req.method == 'GET':
container_size = self.get_container_size( container_size = self.get_container_size(req.environ)
account_name, container_name)
container_rate = get_maxrate( container_rate = get_maxrate(
self.container_listing_ratelimits, container_size) self.container_listing_ratelimits, container_size)
if container_rate: if container_rate:

37
test/unit/common/middleware/test_ratelimit.py
View File

@ -189,7 +189,7 @@ class TestRateLimit(unittest.TestCase):
the_app = ratelimit.filter_factory(conf_dict)(FakeApp()) the_app = ratelimit.filter_factory(conf_dict)(FakeApp())
the_app.memcache_client = fake_memcache the_app.memcache_client = fake_memcache
req = lambda: None req = lambda: None
req.environ = {} req.environ = {'swift.cache': fake_memcache, 'PATH_INFO': '/v1/a/c/o'}
with mock.patch('swift.common.middleware.ratelimit.get_account_info', with mock.patch('swift.common.middleware.ratelimit.get_account_info',
lambda *args, **kwargs: {}): lambda *args, **kwargs: {}):
req.method = 'DELETE' req.method = 'DELETE'
@ -243,7 +243,7 @@ class TestRateLimit(unittest.TestCase):
the_app.memcache_client = fake_memcache the_app.memcache_client = fake_memcache
req = lambda: None req = lambda: None
req.method = 'PUT' req.method = 'PUT'
req.environ = {} req.environ = {'PATH_INFO': '/v1/a/c/o', 'swift.cache': fake_memcache}
with mock.patch('swift.common.middleware.ratelimit.get_account_info', with mock.patch('swift.common.middleware.ratelimit.get_account_info',
lambda *args, **kwargs: {}): lambda *args, **kwargs: {}):
tuples = the_app.get_ratelimitable_key_tuples(req, 'a', 'c', 'o') tuples = the_app.get_ratelimitable_key_tuples(req, 'a', 'c', 'o')
@ -255,20 +255,23 @@ class TestRateLimit(unittest.TestCase):
conf_dict = {'account_ratelimit': current_rate} conf_dict = {'account_ratelimit': current_rate}
self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp())
ratelimit.http_connect = mock_http_connect(204) ratelimit.http_connect = mock_http_connect(204)
with mock.patch('swift.common.middleware.ratelimit.get_account_info', with mock.patch('swift.common.middleware.ratelimit.get_container_info',
lambda *args, **kwargs: {}): lambda *args, **kwargs: {}):
for meth, exp_time in [ with mock.patch(
('DELETE', 9.8), ('GET', 0), ('POST', 0), ('PUT', 9.8)]: 'swift.common.middleware.ratelimit.get_account_info',
req = Request.blank('/v/a%s/c' % meth) lambda *args, **kwargs: {}):
req.method = meth for meth, exp_time in [('DELETE', 9.8), ('GET', 0),
req.environ['swift.cache'] = FakeMemcache() ('POST', 0), ('PUT', 9.8)]:
make_app_call = lambda: self.test_ratelimit(req.environ, req = Request.blank('/v/a%s/c' % meth)
start_response) req.method = meth
begin = time.time() req.environ['swift.cache'] = FakeMemcache()
self._run(make_app_call, num_calls, current_rate, make_app_call = lambda: self.test_ratelimit(req.environ,
check_time=bool(exp_time)) start_response)
self.assertEquals(round(time.time() - begin, 1), exp_time) begin = time.time()
self._reset_time() self._run(make_app_call, num_calls, current_rate,
check_time=bool(exp_time))
self.assertEquals(round(time.time() - begin, 1), exp_time)
self._reset_time()
def test_ratelimit_set_incr(self): def test_ratelimit_set_incr(self):
current_rate = 5 current_rate = 5
@ -403,7 +406,7 @@ class TestRateLimit(unittest.TestCase):
req.method = 'PUT' req.method = 'PUT'
req.environ['swift.cache'] = FakeMemcache() req.environ['swift.cache'] = FakeMemcache()
req.environ['swift.cache'].set( req.environ['swift.cache'].set(
ratelimit.get_container_memcache_key('a', 'c'), get_container_memcache_key('a', 'c'),
{'container_size': 1}) {'container_size': 1})
time_override = [0, 0, 0, 0, None] time_override = [0, 0, 0, 0, None]
@ -437,7 +440,7 @@ class TestRateLimit(unittest.TestCase):
req.method = 'GET' req.method = 'GET'
req.environ['swift.cache'] = FakeMemcache() req.environ['swift.cache'] = FakeMemcache()
req.environ['swift.cache'].set( req.environ['swift.cache'].set(
ratelimit.get_container_memcache_key('a', 'c'), get_container_memcache_key('a', 'c'),
{'container_size': 1}) {'container_size': 1})
time_override = [0, 0, 0, 0, None] time_override = [0, 0, 0, 0, None]