openstack/swift
Code Issues Proposed changes

Fix when rate_limit_after_segment kicks in.

Browse Source 
If rate_limit_after_segment was 10 in the proxy config, then after 10
segments were coughed up by _load_next_segment() with no calls to
sleep(), the 11th segment would not trigger a sleep() call.  The 12th
segment triggered a sleep(0) call, but it was only after the 13th
segment was loaded that an actual rate-limiting (non-zero) sleep got
called.

With this patch, a rate_limit_after_segment of 10 will start sleeping
the correct amount after the 11th segment.

Updated proxy-server.conf-sample with rate_limit_after_segment and
rate_limit_segments_per_sec.

Change-Id: I937c366996e6d6ab47c614d6db470e3be9657c07
This commit is contained in:
Darrell Bishop 2012-08-20 22:51:46 -07:00
parent d8c02dccc0
commit 1a6c42fccd
3 changed files with 47 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
etc/proxy-server.conf-sample
View File

@ -67,12 +67,18 @@ use = egg:swift#proxy
# This is a comma separated list of account hashes that ignore the
# max_containers_per_account cap.
# max_containers_whitelist =
# comma separated list of Host headers the proxy will be deny requests to
# Comma separated list of Host headers to which the proxy will deny requests.
# deny_host_headers =
# prefix used when automatically creating accounts
# Prefix used when automatically creating accounts.
# auto_create_account_prefix = .
# depth of the proxy put queue
# Depth of the proxy put queue.
# put_queue_depth = 10
# Start rate-limiting object segment serving after the Nth segment of a
# segmented object.
# rate_limit_after_segment = 10
# Once segment rate-limiting kicks in for an object, limit segments served
# to N per second.
# rate_limit_segments_per_sec = 1
[filter:tempauth]
use = egg:swift#tempauth

6
swift/proxy/controllers/obj.py
View File

@ -80,7 +80,7 @@ class SegmentedIterable(object):
self.controller = controller
self.container = container
self.listing = iter(listing)
self.segment = -1
self.segment = 0
self.segment_dict = None
self.segment_peek = None
self.seek = 0
@ -114,8 +114,8 @@ class SegmentedIterable(object):
self.seek = 0
if self.segment > self.controller.app.rate_limit_after_segment:
sleep(max(self.next_get_time - time.time(), 0))
self.next_get_time = time.time() + \
1.0 / self.controller.app.rate_limit_segments_per_sec
self.next_get_time = time.time() + \
1.0 / self.controller.app.rate_limit_segments_per_sec
shuffle(nodes)
resp = self.controller.GETorHEAD_base(req, _('Object'), partition,
self.controller.iter_nodes(partition, nodes,

43
test/unit/proxy/test_server.py
View File

@ -325,7 +325,6 @@ def save_globals():
yield True
finally:
proxy_server.Controller.account_info = orig_account_info
proxy_server.http_connect = orig_http_connect
swift.proxy.controllers.base.http_connect = orig_http_connect
swift.proxy.controllers.obj.http_connect = orig_http_connect
swift.proxy.controllers.account.http_connect = orig_http_connect
@ -334,7 +333,6 @@ def save_globals():
def set_http_connect(*args, **kwargs):
new_connect = fake_http_connect(*args, **kwargs)
proxy_server.http_connect = new_connect
swift.proxy.controllers.base.http_connect = new_connect
swift.proxy.controllers.obj.http_connect = new_connect
swift.proxy.controllers.account.http_connect = new_connect
@ -480,8 +478,7 @@ class TestController(unittest.TestCase):
self.assertEquals(None, count)
self.memcache.store = {}
proxy_server.http_connect = \
fake_http_connect(404, 404, 404, 403, 403, 403)
set_http_connect(404, 404, 404, 403, 403, 403)
exc = None
partition, nodes, count = \
self.controller.account_info(self.account, autocreate=True)
@ -489,8 +486,7 @@ class TestController(unittest.TestCase):
self.assertEquals(None, count)
self.memcache.store = {}
proxy_server.http_connect = \
fake_http_connect(404, 404, 404, 409, 409, 409)
set_http_connect(404, 404, 404, 409, 409, 409)
exc = None
partition, nodes, count = \
self.controller.account_info(self.account, autocreate=True)
@ -4044,8 +4040,8 @@ class FakeObjectController(object):
self.trans_id = 'tx1'
self.object_ring = FakeRing()
self.node_timeout = 1
self.rate_limit_after_segment = 10
self.rate_limit_segments_per_sec = 1
self.rate_limit_after_segment = 3
self.rate_limit_segments_per_sec = 2
def exception(self, *args):
self.exception_args = args
@ -4113,6 +4109,37 @@ class TestSegmentedIterable(unittest.TestCase):
data = ''.join(segit.segment_iter)
self.assertEquals(data, '22')
def test_load_next_segment_rate_limiting(self):
sleep_calls = []
def _stub_sleep(sleepy_time):
sleep_calls.append(sleepy_time)
orig_sleep = swift.proxy.controllers.obj.sleep
try:
swift.proxy.controllers.obj.sleep = _stub_sleep
segit = SegmentedIterable(
self.controller, 'lc', [
{'name': 'o1'}, {'name': 'o2'}, {'name': 'o3'},
{'name': 'o4'}, {'name': 'o5'}])
# rate_limit_after_segment == 3, so the first 3 segments should invoke
# no sleeping.
for _ in xrange(3):
segit._load_next_segment()
self.assertEquals([], sleep_calls)
self.assertEquals(self.controller.GETorHEAD_base_args[4], '/a/lc/o3')
# Loading of next (4th) segment starts rate-limiting.
segit._load_next_segment()
self.assertAlmostEqual(0.5, sleep_calls[0], places=2)
self.assertEquals(self.controller.GETorHEAD_base_args[4], '/a/lc/o4')
sleep_calls = []
segit._load_next_segment()
self.assertAlmostEqual(0.5, sleep_calls[0], places=2)
self.assertEquals(self.controller.GETorHEAD_base_args[4], '/a/lc/o5')
finally:
swift.proxy.controllers.obj.sleep = orig_sleep
def test_load_next_segment_with_two_segments_skip_first(self):
segit = SegmentedIterable(self.controller, 'lc', [{'name':
'o1'}, {'name': 'o2'}])