From b8f73a171d8d12c82a92435022d06755cd8a605b Mon Sep 17 00:00:00 2001 From: Peter Portante Date: Wed, 26 Mar 2014 22:10:29 -0400 Subject: [PATCH] Move like unit tests together; add comments Some simple code movement to move the utils.ratelimit_sleep() unit tests together so that they can be viewed all at once. We also add some comments to document the behavior of utils.ratelimit_sleep(); small modification to max_rate parameter checking to match intended use. Change-Id: I3b11acfb6634d16a4b3594dba8dbc7a2d3ee8d1a --- swift/common/utils.py | 25 ++++++++++++++++++++----- test/unit/common/test_utils.py | 28 +++++++++++++++++----------- 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/swift/common/utils.py b/swift/common/utils.py index be76ddb6c1..5281be9410 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -1754,26 +1754,41 @@ def ratelimit_sleep(running_time, max_rate, incr_by=1, rate_buffer=5): as eventlet.sleep() does involve some overhead. Returns running_time that should be used for subsequent calls. - :param running_time: the running time of the next allowable request. Best - to start at zero. + :param running_time: the running time in milliseconds of the next + allowable request. Best to start at zero. :param max_rate: The maximum rate per second allowed for the process. :param incr_by: How much to increment the counter. Useful if you want to ratelimit 1024 bytes/sec and have differing sizes - of requests. Must be >= 0. + of requests. Must be > 0 to engage rate-limiting + behavior. :param rate_buffer: Number of seconds the rate counter can drop and be allowed to catch up (at a faster than listed rate). A larger number will result in larger spikes in rate - but better average accuracy. + but better average accuracy. Must be > 0 to engage + rate-limiting behavior. ''' - if not max_rate or incr_by <= 0: + if max_rate <= 0 or incr_by <= 0: return running_time + + # 1,000 milliseconds = 1 second clock_accuracy = 1000.0 + + # Convert seconds to milliseconds now = time.time() * clock_accuracy + + # Calculate time per request in milliseconds time_per_request = clock_accuracy * (float(incr_by) / max_rate) + + # Convert rate_buffer to milliseconds and compare if now - running_time > rate_buffer * clock_accuracy: running_time = now elif running_time - now > time_per_request: + # Convert diff back to a floating point number of seconds and sleep eventlet.sleep((running_time - now) / clock_accuracy) + + # Return the absolute time for the next interval in milliseconds; note + # that time could have passed well beyond that point, but the next call + # will catch that and skip the sleep. return running_time + time_per_request diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 4455a5f727..63033209b4 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -1075,6 +1075,12 @@ log_name = %(yarr)s''' reset_loggers() def test_ratelimit_sleep(self): + running_time = 0 + start = time.time() + for i in range(100): + running_time = utils.ratelimit_sleep(running_time, -5) + self.assertTrue(abs((time.time() - start) * 100) < 1) + running_time = 0 start = time.time() for i in range(100): @@ -1100,6 +1106,17 @@ log_name = %(yarr)s''' total += i self.assertTrue(abs(50 - (time.time() - start) * 100) < 10) + def test_ratelimit_sleep_with_sleep(self): + running_time = 0 + start = time.time() + sleeps = [0] * 7 + [.2] * 3 + [0] * 30 + for i in sleeps: + running_time = utils.ratelimit_sleep(running_time, 40, + rate_buffer=1) + time.sleep(i) + # make sure it's accurate to 10th of a second + self.assertTrue(abs(100 - (time.time() - start) * 100) < 10) + def test_urlparse(self): parsed = utils.urlparse('http://127.0.0.1/') self.assertEquals(parsed.scheme, 'http') @@ -1122,17 +1139,6 @@ log_name = %(yarr)s''' parsed = utils.urlparse('www.example.com') self.assertEquals(parsed.hostname, '') - def test_ratelimit_sleep_with_sleep(self): - running_time = 0 - start = time.time() - sleeps = [0] * 7 + [.2] * 3 + [0] * 30 - for i in sleeps: - running_time = utils.ratelimit_sleep(running_time, 40, - rate_buffer=1) - time.sleep(i) - # make sure it's accurate to 10th of a second - self.assertTrue(abs(100 - (time.time() - start) * 100) < 10) - def test_search_tree(self): # file match & ext miss with temptree(['asdf.conf', 'blarg.conf', 'asdf.cfg']) as t: