From 7f636a557296ecc6ae4727700cfcf9f82573bd16 Mon Sep 17 00:00:00 2001 From: Samuel Merritt Date: Mon, 30 Nov 2015 18:06:09 -0800 Subject: [PATCH] Allow smaller segments in static large objects The addition of range support for SLO segments (commit 25d5e68) required the range size to be at least the SLO minimum segment size (default 1 MiB). However, if you're doing something like assembling a video of short clips out of a larger one, then you might not need a full 1 MiB. The reason for the 1 MiB restriction was to protect Swift from resource overconsumption. It takes CPU, RAM, and internal bandwidth to connect to an object server, so it's much cheaper to serve a 10 GiB SLO if it has 10 MiB segments than if it has 10 B segments. Instead of a strict limit, now we apply ratelimiting to small segments. The threshold for "small" is configurable and defaults to 1 MiB. SLO segments may now be as small as 1 byte. If a client makes SLOs as before, it'll still be able to download the objects as fast as Swift can serve them. However, a SLO with a lot of small ranges or segments will be slowed down to avoid resource overconsumption. This is similar to how DLOs work, except that DLOs ratelimit *every* segment, not just small ones. UpgradeImpact For operators: if your cluster has enabled ratelimiting for SLO, you will want to set rate_limit_under_size to a large number prior to upgrade. This will preserve your existing behavior of ratelimiting all SLO segments. 5368709123 is a good value, as that's 1 greater than the default max object size. Alternately, hold down the 9 key until you get bored. If your cluster has not enabled ratelimiting for SLO (the default), no action is needed. Change-Id: Id1ff7742308ed816038a5c44ec548afa26612b95 --- etc/proxy-server.conf-sample | 9 +- swift/common/middleware/slo.py | 59 +++-- swift/common/utils.py | 19 +- test/unit/common/middleware/helpers.py | 9 +- test/unit/common/middleware/test_slo.py | 323 ++++++++++++++++-------- test/unit/common/test_utils.py | 20 ++ 6 files changed, 287 insertions(+), 152 deletions(-) diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 464705d240..bd421e5030 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -622,14 +622,17 @@ use = egg:swift#bulk use = egg:swift#slo # max_manifest_segments = 1000 # max_manifest_size = 2097152 -# min_segment_size = 1048576 -# Start rate-limiting SLO segment serving after the Nth segment of a +# +# Rate limiting applies only to segments smaller than this size (bytes). +# rate_limit_under_size = 1048576 +# +# Start rate-limiting SLO segment serving after the Nth small segment of a # segmented object. # rate_limit_after_segment = 10 # # Once segment rate-limiting kicks in for an object, limit segments served # to N per second. 0 means no rate-limiting. -# rate_limit_segments_per_sec = 0 +# rate_limit_segments_per_sec = 1 # # Time limit on GET requests (seconds) # max_get_time = 86400 diff --git a/swift/common/middleware/slo.py b/swift/common/middleware/slo.py index 048d8b5add..37f9a1fad2 100644 --- a/swift/common/middleware/slo.py +++ b/swift/common/middleware/slo.py @@ -57,12 +57,11 @@ The format of the list will be: "range": "1048576-2097151"}, ...] The number of object segments is limited to a configurable amount, default -1000. Each segment, except for the final one, must be at least 1 megabyte -(configurable). On upload, the middleware will head every segment passed in to -verify: +1000. Each segment must be at least 1 byte. On upload, the middleware will +head every segment passed in to verify: 1. the segment exists (i.e. the HEAD was successful); - 2. the segment meets minimum size requirements (if not the last segment); + 2. the segment meets minimum size requirements; 3. if the user provided a non-null etag, the etag matches; 4. if the user provided a non-null size_bytes, the size_bytes matches; and 5. if the user provided a range, it is a singular, syntactically correct range @@ -121,8 +120,9 @@ finally bytes 2095104 through 2097152 (i.e., the last 2048 bytes) of .. note:: - The minimum sized range is min_segment_size, which by - default is 1048576 (1MB). + + The minimum sized range is 1 byte. This is the same as the minimum + segment size. ------------------------- @@ -221,7 +221,7 @@ from swift.common.middleware.bulk import get_response_body, \ ACCEPTABLE_FORMATS, Bulk -DEFAULT_MIN_SEGMENT_SIZE = 1024 * 1024 # 1 MiB +DEFAULT_RATE_LIMIT_UNDER_SIZE = 1024 * 1024 # 1 MiB DEFAULT_MAX_MANIFEST_SEGMENTS = 1000 DEFAULT_MAX_MANIFEST_SIZE = 1024 * 1024 * 2 # 2 MiB @@ -231,7 +231,7 @@ OPTIONAL_SLO_KEYS = set(['range']) ALLOWED_SLO_KEYS = REQUIRED_SLO_KEYS | OPTIONAL_SLO_KEYS -def parse_and_validate_input(req_body, req_path, min_segment_size): +def parse_and_validate_input(req_body, req_path): """ Given a request body, parses it and returns a list of dictionaries. @@ -269,7 +269,6 @@ def parse_and_validate_input(req_body, req_path, min_segment_size): vrs, account, _junk = split_path(req_path, 3, 3, True) errors = [] - num_segs = len(parsed_data) for seg_index, seg_dict in enumerate(parsed_data): if not isinstance(seg_dict, dict): errors.append("Index %d: not a JSON object" % seg_index) @@ -315,10 +314,10 @@ def parse_and_validate_input(req_body, req_path, min_segment_size): except (TypeError, ValueError): errors.append("Index %d: invalid size_bytes" % seg_index) continue - if (seg_size < min_segment_size and seg_index < num_segs - 1): - errors.append("Index %d: too small; each segment, except " - "the last, must be at least %d bytes." - % (seg_index, min_segment_size)) + if seg_size < 1: + errors.append("Index %d: too small; each segment must be " + "at least 1 byte." + % (seg_index,)) continue obj_path = '/'.join(['', vrs, account, seg_dict['path'].lstrip('/')]) @@ -662,10 +661,17 @@ class SloGetContext(WSGIContext): plain_listing_iter = self._segment_listing_iterator( req, ver, account, segments) + def is_small_segment((seg_dict, start_byte, end_byte)): + start = 0 if start_byte is None else start_byte + end = int(seg_dict['bytes']) - 1 if end_byte is None else end_byte + is_small = (end - start + 1) < self.slo.rate_limit_under_size + return is_small + ratelimited_listing_iter = RateLimitedIterator( plain_listing_iter, self.slo.rate_limit_segments_per_sec, - limit_after=self.slo.rate_limit_after_segment) + limit_after=self.slo.rate_limit_after_segment, + ratelimit_if=is_small_segment) # self._segment_listing_iterator gives us 3-tuples of (segment dict, # start byte, end byte), but SegmentedIterable wants (obj path, etag, @@ -716,7 +722,7 @@ class StaticLargeObject(object): :param conf: The configuration dict for the middleware. """ - def __init__(self, app, conf, min_segment_size=DEFAULT_MIN_SEGMENT_SIZE, + def __init__(self, app, conf, max_manifest_segments=DEFAULT_MAX_MANIFEST_SEGMENTS, max_manifest_size=DEFAULT_MAX_MANIFEST_SIZE): self.conf = conf @@ -724,12 +730,13 @@ class StaticLargeObject(object): self.logger = get_logger(conf, log_route='slo') self.max_manifest_segments = max_manifest_segments self.max_manifest_size = max_manifest_size - self.min_segment_size = min_segment_size self.max_get_time = int(self.conf.get('max_get_time', 86400)) + self.rate_limit_under_size = int(self.conf.get( + 'rate_limit_under_size', DEFAULT_RATE_LIMIT_UNDER_SIZE)) self.rate_limit_after_segment = int(self.conf.get( 'rate_limit_after_segment', '10')) self.rate_limit_segments_per_sec = int(self.conf.get( - 'rate_limit_segments_per_sec', '0')) + 'rate_limit_segments_per_sec', '1')) self.bulk_deleter = Bulk(app, {}, logger=self.logger) def handle_multipart_get_or_head(self, req, start_response): @@ -783,7 +790,7 @@ class StaticLargeObject(object): raise HTTPLengthRequired(request=req) parsed_data = parse_and_validate_input( req.body_file.read(self.max_manifest_size), - req.path, self.min_segment_size) + req.path) problem_segments = [] if len(parsed_data) > self.max_manifest_segments: @@ -812,6 +819,7 @@ class StaticLargeObject(object): new_env['CONTENT_LENGTH'] = 0 new_env['HTTP_USER_AGENT'] = \ '%s MultipartPUT' % req.environ.get('HTTP_USER_AGENT') + if obj_path != last_obj_path: last_obj_path = obj_path head_seg_resp = \ @@ -840,12 +848,10 @@ class StaticLargeObject(object): seg_dict['range'] = '%d-%d' % (rng[0], rng[1] - 1) segment_length = rng[1] - rng[0] - if segment_length < self.min_segment_size and \ - index < len(parsed_data) - 1: + if segment_length < 1: problem_segments.append( [quote(obj_name), - 'Too small; each segment, except the last, must be ' - 'at least %d bytes.' % self.min_segment_size]) + 'Too small; each segment must be at least 1 byte.']) total_size += segment_length if seg_dict['size_bytes'] is not None and \ seg_dict['size_bytes'] != head_seg_resp.content_length: @@ -1045,18 +1051,17 @@ def filter_factory(global_conf, **local_conf): DEFAULT_MAX_MANIFEST_SEGMENTS)) max_manifest_size = int(conf.get('max_manifest_size', DEFAULT_MAX_MANIFEST_SIZE)) - min_segment_size = int(conf.get('min_segment_size', - DEFAULT_MIN_SEGMENT_SIZE)) register_swift_info('slo', max_manifest_segments=max_manifest_segments, max_manifest_size=max_manifest_size, - min_segment_size=min_segment_size) + # this used to be configurable; report it as 1 for + # clients that might still care + min_segment_size=1) def slo_filter(app): return StaticLargeObject( app, conf, max_manifest_segments=max_manifest_segments, - max_manifest_size=max_manifest_size, - min_segment_size=min_segment_size) + max_manifest_size=max_manifest_size) return slo_filter diff --git a/swift/common/utils.py b/swift/common/utils.py index d6cc5d7afb..4e597d1b26 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -1041,22 +1041,27 @@ class RateLimitedIterator(object): this many elements; default is 0 (rate limit immediately) """ - def __init__(self, iterable, elements_per_second, limit_after=0): + def __init__(self, iterable, elements_per_second, limit_after=0, + ratelimit_if=lambda _junk: True): self.iterator = iter(iterable) self.elements_per_second = elements_per_second self.limit_after = limit_after self.running_time = 0 + self.ratelimit_if = ratelimit_if def __iter__(self): return self def next(self): - if self.limit_after > 0: - self.limit_after -= 1 - else: - self.running_time = ratelimit_sleep(self.running_time, - self.elements_per_second) - return next(self.iterator) + next_value = next(self.iterator) + + if self.ratelimit_if(next_value): + if self.limit_after > 0: + self.limit_after -= 1 + else: + self.running_time = ratelimit_sleep(self.running_time, + self.elements_per_second) + return next_value class GreenthreadSafeIterator(object): diff --git a/test/unit/common/middleware/helpers.py b/test/unit/common/middleware/helpers.py index bc6ad50fdd..1387a773b4 100644 --- a/test/unit/common/middleware/helpers.py +++ b/test/unit/common/middleware/helpers.py @@ -56,7 +56,7 @@ class FakeSwift(object): self.container_ring = FakeRing() self.get_object_ring = lambda policy_index: FakeRing() - def _get_response(self, method, path): + def _find_response(self, method, path): resp = self._responses[(method, path)] if isinstance(resp, list): try: @@ -84,16 +84,17 @@ class FakeSwift(object): self.swift_sources.append(env.get('swift.source')) try: - resp_class, raw_headers, body = self._get_response(method, path) + resp_class, raw_headers, body = self._find_response(method, path) headers = swob.HeaderKeyDict(raw_headers) except KeyError: if (env.get('QUERY_STRING') and (method, env['PATH_INFO']) in self._responses): - resp_class, raw_headers, body = self._get_response( + resp_class, raw_headers, body = self._find_response( method, env['PATH_INFO']) headers = swob.HeaderKeyDict(raw_headers) elif method == 'HEAD' and ('GET', path) in self._responses: - resp_class, raw_headers, body = self._get_response('GET', path) + resp_class, raw_headers, body = self._find_response( + 'GET', path) body = None headers = swob.HeaderKeyDict(raw_headers) elif method == 'GET' and obj and path in self.uploaded: diff --git a/test/unit/common/middleware/test_slo.py b/test/unit/common/middleware/test_slo.py index 32d49547d4..897bf551f9 100644 --- a/test/unit/common/middleware/test_slo.py +++ b/test/unit/common/middleware/test_slo.py @@ -55,8 +55,8 @@ def md5hex(s): class SloTestCase(unittest.TestCase): def setUp(self): self.app = FakeSwift() - self.slo = slo.filter_factory({})(self.app) - self.slo.min_segment_size = 1 + slo_conf = {'rate_limit_under_size': '0'} + self.slo = slo.filter_factory(slo_conf)(self.app) self.slo.logger = self.app.logger def call_app(self, req, app=None, expect_exception=False): @@ -120,18 +120,14 @@ class TestSloMiddleware(SloTestCase): resp.startswith('X-Static-Large-Object is a reserved header')) def _put_bogus_slo(self, manifest_text, - manifest_path='/v1/a/c/the-manifest', - min_segment_size=1): + manifest_path='/v1/a/c/the-manifest'): with self.assertRaises(HTTPException) as catcher: - slo.parse_and_validate_input(manifest_text, manifest_path, - min_segment_size) + slo.parse_and_validate_input(manifest_text, manifest_path) self.assertEqual(400, catcher.exception.status_int) return catcher.exception.body - def _put_slo(self, manifest_text, manifest_path='/v1/a/c/the-manifest', - min_segment_size=1): - return slo.parse_and_validate_input(manifest_text, manifest_path, - min_segment_size) + def _put_slo(self, manifest_text, manifest_path='/v1/a/c/the-manifest'): + return slo.parse_and_validate_input(manifest_text, manifest_path) def test_bogus_input(self): self.assertEqual('Manifest must be valid JSON.\n', @@ -248,19 +244,18 @@ class TestSloMiddleware(SloTestCase): def test_bogus_input_undersize_segment(self): self.assertEqual( - "Index 1: too small; each segment, except the last, " - "must be at least 1000 bytes.\n" - "Index 2: too small; each segment, except the last, " - "must be at least 1000 bytes.\n", + "Index 1: too small; each segment " + "must be at least 1 byte.\n" + "Index 2: too small; each segment " + "must be at least 1 byte.\n", self._put_bogus_slo( json.dumps([ - {'path': u'/c/s1', 'etag': 'a', 'size_bytes': 1000}, - {'path': u'/c/s2', 'etag': 'b', 'size_bytes': 999}, - {'path': u'/c/s3', 'etag': 'c', 'size_bytes': 998}, + {'path': u'/c/s1', 'etag': 'a', 'size_bytes': 1}, + {'path': u'/c/s2', 'etag': 'b', 'size_bytes': 0}, + {'path': u'/c/s3', 'etag': 'c', 'size_bytes': 0}, # No error for this one since size_bytes is unspecified {'path': u'/c/s4', 'etag': 'd', 'size_bytes': None}, - {'path': u'/c/s5', 'etag': 'e', 'size_bytes': 996}]), - min_segment_size=1000)) + {'path': u'/c/s5', 'etag': 'e', 'size_bytes': 1000}]))) def test_valid_input(self): data = json.dumps( @@ -268,19 +263,19 @@ class TestSloMiddleware(SloTestCase): 'size_bytes': 100}]) self.assertEqual( '/cont/object', - slo.parse_and_validate_input(data, '/v1/a/cont/man', 1)[0]['path']) + slo.parse_and_validate_input(data, '/v1/a/cont/man')[0]['path']) data = json.dumps( [{'path': '/cont/object', 'etag': 'etagoftheobjectsegment', 'size_bytes': 100, 'range': '0-40'}]) - parsed = slo.parse_and_validate_input(data, '/v1/a/cont/man', 1) + parsed = slo.parse_and_validate_input(data, '/v1/a/cont/man') self.assertEqual('/cont/object', parsed[0]['path']) self.assertEqual([(0, 40)], parsed[0]['range'].ranges) data = json.dumps( [{'path': '/cont/object', 'etag': 'etagoftheobjectsegment', 'size_bytes': None, 'range': '0-40'}]) - parsed = slo.parse_and_validate_input(data, '/v1/a/cont/man', 1) + parsed = slo.parse_and_validate_input(data, '/v1/a/cont/man') self.assertEqual('/cont/object', parsed[0]['path']) self.assertEqual(None, parsed[0]['size_bytes']) self.assertEqual([(0, 40)], parsed[0]['range'].ranges) @@ -316,6 +311,11 @@ class TestSloPutManifest(SloTestCase): swob.HTTPOk, {'Content-Length': '10', 'Etag': 'etagoftheobjectsegment'}, None) + self.app.register( + 'HEAD', '/v1/AUTH_test/cont/empty_object', + swob.HTTPOk, + {'Content-Length': '0', 'Etag': 'etagoftheobjectsegment'}, + None) self.app.register( 'HEAD', u'/v1/AUTH_test/cont/あ_1', swob.HTTPOk, @@ -340,11 +340,17 @@ class TestSloPutManifest(SloTestCase): {'Content-Length': '2', 'Etag': 'b', 'Last-Modified': 'Fri, 01 Feb 2012 20:38:36 GMT'}, None) + + _manifest_json = json.dumps( + [{'name': '/checktest/a_5', 'hash': md5hex("a" * 5), + 'content_type': 'text/plain', 'bytes': '5'}]) self.app.register( 'GET', '/v1/AUTH_test/checktest/slob', swob.HTTPOk, - {'X-Static-Large-Object': 'true', 'Etag': 'slob-etag'}, - None) + {'X-Static-Large-Object': 'true', 'Etag': 'slob-etag', + 'Content-Type': 'cat/picture;swift_bytes=12345', + 'Content-Length': len(_manifest_json)}, + _manifest_json) self.app.register( 'PUT', '/v1/AUTH_test/checktest/man_3', swob.HTTPCreated, {}, None) @@ -367,21 +373,6 @@ class TestSloPutManifest(SloTestCase): pass self.assertEqual(e.status_int, 413) - with patch.object(self.slo, 'min_segment_size', 1000): - test_json_data_2obj = json.dumps( - [{'path': '/cont/small_object1', - 'etag': 'etagoftheobjectsegment', - 'size_bytes': 10}, - {'path': '/cont/small_object2', - 'etag': 'etagoftheobjectsegment', - 'size_bytes': 10}]) - req = Request.blank('/v1/a/c/o', body=test_json_data_2obj) - try: - self.slo.handle_multipart_put(req, fake_start_response) - except HTTPException as e: - pass - self.assertEqual(e.status_int, 400) - req = Request.blank('/v1/a/c/o', headers={'X-Copy-From': 'lala'}) try: self.slo.handle_multipart_put(req, fake_start_response) @@ -411,49 +402,29 @@ class TestSloPutManifest(SloTestCase): self.slo(req.environ, my_fake_start_response) self.assertTrue('X-Static-Large-Object' in req.headers) - def test_handle_multipart_put_success_allow_small_last_segment(self): - with patch.object(self.slo, 'min_segment_size', 50): - test_json_data = json.dumps([{'path': '/cont/object', - 'etag': 'etagoftheobjectsegment', - 'size_bytes': 100}, - {'path': '/cont/small_object', - 'etag': 'etagoftheobjectsegment', - 'size_bytes': 10}]) - req = Request.blank( - '/v1/AUTH_test/c/man?multipart-manifest=put', - environ={'REQUEST_METHOD': 'PUT'}, headers={'Accept': 'test'}, - body=test_json_data) - self.assertTrue('X-Static-Large-Object' not in req.headers) - self.slo(req.environ, fake_start_response) - self.assertTrue('X-Static-Large-Object' in req.headers) + def test_handle_multipart_put_disallow_empty_first_segment(self): + test_json_data = json.dumps([{'path': '/cont/object', + 'etag': 'etagoftheobjectsegment', + 'size_bytes': 0}, + {'path': '/cont/small_object', + 'etag': 'etagoftheobjectsegment', + 'size_bytes': 100}]) + req = Request.blank('/v1/a/c/o', body=test_json_data) + with self.assertRaises(HTTPException) as catcher: + self.slo.handle_multipart_put(req, fake_start_response) + self.assertEqual(catcher.exception.status_int, 400) - def test_handle_multipart_put_success_allow_only_one_small_segment(self): - with patch.object(self.slo, 'min_segment_size', 50): - test_json_data = json.dumps([{'path': '/cont/small_object', - 'etag': 'etagoftheobjectsegment', - 'size_bytes': 10}]) - req = Request.blank( - '/v1/AUTH_test/c/man?multipart-manifest=put', - environ={'REQUEST_METHOD': 'PUT'}, headers={'Accept': 'test'}, - body=test_json_data) - self.assertTrue('X-Static-Large-Object' not in req.headers) - self.slo(req.environ, fake_start_response) - self.assertTrue('X-Static-Large-Object' in req.headers) - - def test_handle_multipart_put_disallow_small_first_segment(self): - with patch.object(self.slo, 'min_segment_size', 50): - test_json_data = json.dumps([{'path': '/cont/object', - 'etag': 'etagoftheobjectsegment', - 'size_bytes': 10}, - {'path': '/cont/small_object', - 'etag': 'etagoftheobjectsegment', - 'size_bytes': 100}]) - req = Request.blank('/v1/a/c/o', body=test_json_data) - try: - self.slo.handle_multipart_put(req, fake_start_response) - except HTTPException as e: - pass - self.assertEqual(e.status_int, 400) + def test_handle_multipart_put_disallow_empty_last_segment(self): + test_json_data = json.dumps([{'path': '/cont/object', + 'etag': 'etagoftheobjectsegment', + 'size_bytes': 100}, + {'path': '/cont/small_object', + 'etag': 'etagoftheobjectsegment', + 'size_bytes': 0}]) + req = Request.blank('/v1/a/c/o', body=test_json_data) + with self.assertRaises(HTTPException) as catcher: + self.slo.handle_multipart_put(req, fake_start_response) + self.assertEqual(catcher.exception.status_int, 400) def test_handle_multipart_put_success_unicode(self): test_json_data = json.dumps([{'path': u'/cont/object\u2661', @@ -543,7 +514,7 @@ class TestSloPutManifest(SloTestCase): {'path': '/checktest/badreq', 'etag': 'a', 'size_bytes': '1'}, {'path': '/checktest/b_2', 'etag': 'not-b', 'size_bytes': '2'}, {'path': '/checktest/slob', 'etag': 'not-slob', - 'size_bytes': '2'}]) + 'size_bytes': '12345'}]) req = Request.blank( '/v1/AUTH_test/checktest/man?multipart-manifest=put', environ={'REQUEST_METHOD': 'PUT'}, @@ -553,6 +524,7 @@ class TestSloPutManifest(SloTestCase): status, headers, body = self.call_slo(req) self.assertEqual(self.app.call_count, 5) errors = json.loads(body)['Errors'] + self.assertEqual(len(errors), 5) self.assertEqual(errors[0][0], '/checktest/a_1') self.assertEqual(errors[0][1], 'Size Mismatch') @@ -587,35 +559,33 @@ class TestSloPutManifest(SloTestCase): self.assertEqual(2, manifest_data[1]['bytes']) def test_handle_multipart_put_skip_size_check_still_uses_min_size(self): - with patch.object(self.slo, 'min_segment_size', 50): - test_json_data = json.dumps([{'path': '/cont/small_object', - 'etag': 'etagoftheobjectsegment', - 'size_bytes': None}, - {'path': '/cont/small_object', - 'etag': 'etagoftheobjectsegment', - 'size_bytes': 100}]) - req = Request.blank('/v1/AUTH_test/c/o', body=test_json_data) - with self.assertRaises(HTTPException) as cm: - self.slo.handle_multipart_put(req, fake_start_response) - self.assertEqual(cm.exception.status_int, 400) + test_json_data = json.dumps([{'path': '/cont/empty_object', + 'etag': 'etagoftheobjectsegment', + 'size_bytes': None}, + {'path': '/cont/small_object', + 'etag': 'etagoftheobjectsegment', + 'size_bytes': 100}]) + req = Request.blank('/v1/AUTH_test/c/o', body=test_json_data) + with self.assertRaises(HTTPException) as cm: + self.slo.handle_multipart_put(req, fake_start_response) + self.assertEqual(cm.exception.status_int, 400) def test_handle_multipart_put_skip_size_check_no_early_bailout(self): - with patch.object(self.slo, 'min_segment_size', 50): - # The first is too small (it's 10 bytes but min size is 50), and - # the second has a bad etag. Make sure both errors show up in - # the response. - test_json_data = json.dumps([{'path': '/cont/small_object', - 'etag': 'etagoftheobjectsegment', - 'size_bytes': None}, - {'path': '/cont/object2', - 'etag': 'wrong wrong wrong', - 'size_bytes': 100}]) - req = Request.blank('/v1/AUTH_test/c/o', body=test_json_data) - with self.assertRaises(HTTPException) as cm: - self.slo.handle_multipart_put(req, fake_start_response) - self.assertEqual(cm.exception.status_int, 400) - self.assertIn('at least 50 bytes', cm.exception.body) - self.assertIn('Etag Mismatch', cm.exception.body) + # The first is too small (it's 0 bytes), and + # the second has a bad etag. Make sure both errors show up in + # the response. + test_json_data = json.dumps([{'path': '/cont/empty_object', + 'etag': 'etagoftheobjectsegment', + 'size_bytes': None}, + {'path': '/cont/object2', + 'etag': 'wrong wrong wrong', + 'size_bytes': 100}]) + req = Request.blank('/v1/AUTH_test/c/o', body=test_json_data) + with self.assertRaises(HTTPException) as cm: + self.slo.handle_multipart_put(req, fake_start_response) + self.assertEqual(cm.exception.status_int, 400) + self.assertIn('at least 1 byte', cm.exception.body) + self.assertIn('Etag Mismatch', cm.exception.body) def test_handle_multipart_put_skip_etag_check(self): good_data = json.dumps( @@ -1126,6 +1096,46 @@ class TestSloGetManifest(SloTestCase): swob.HTTPOk, {'Content-Length': '20', 'Etag': md5hex('d' * 20)}, 'd' * 20) + self.app.register( + 'GET', '/v1/AUTH_test/gettest/e_25', + swob.HTTPOk, {'Content-Length': '25', + 'Etag': md5hex('e' * 25)}, + 'e' * 25) + self.app.register( + 'GET', '/v1/AUTH_test/gettest/f_30', + swob.HTTPOk, {'Content-Length': '30', + 'Etag': md5hex('f' * 30)}, + 'f' * 30) + self.app.register( + 'GET', '/v1/AUTH_test/gettest/g_35', + swob.HTTPOk, {'Content-Length': '35', + 'Etag': md5hex('g' * 35)}, + 'g' * 35) + self.app.register( + 'GET', '/v1/AUTH_test/gettest/h_40', + swob.HTTPOk, {'Content-Length': '40', + 'Etag': md5hex('h' * 40)}, + 'h' * 40) + self.app.register( + 'GET', '/v1/AUTH_test/gettest/i_45', + swob.HTTPOk, {'Content-Length': '45', + 'Etag': md5hex('i' * 45)}, + 'i' * 45) + self.app.register( + 'GET', '/v1/AUTH_test/gettest/j_50', + swob.HTTPOk, {'Content-Length': '50', + 'Etag': md5hex('j' * 50)}, + 'j' * 50) + self.app.register( + 'GET', '/v1/AUTH_test/gettest/k_55', + swob.HTTPOk, {'Content-Length': '55', + 'Etag': md5hex('k' * 55)}, + 'k' * 55) + self.app.register( + 'GET', '/v1/AUTH_test/gettest/l_60', + swob.HTTPOk, {'Content-Length': '60', + 'Etag': md5hex('l' * 60)}, + 'l' * 60) _bc_manifest_json = json.dumps( [{'name': '/gettest/b_10', 'hash': md5hex('b' * 10), 'bytes': '10', @@ -1156,6 +1166,39 @@ class TestSloGetManifest(SloTestCase): 'Etag': md5(_abcd_manifest_json).hexdigest()}, _abcd_manifest_json) + _abcdefghijkl_manifest_json = json.dumps( + [{'name': '/gettest/a_5', 'hash': md5hex("a" * 5), + 'content_type': 'text/plain', 'bytes': '5'}, + {'name': '/gettest/b_10', 'hash': md5hex("b" * 10), + 'content_type': 'text/plain', 'bytes': '10'}, + {'name': '/gettest/c_15', 'hash': md5hex("c" * 15), + 'content_type': 'text/plain', 'bytes': '15'}, + {'name': '/gettest/d_20', 'hash': md5hex("d" * 20), + 'content_type': 'text/plain', 'bytes': '20'}, + {'name': '/gettest/e_25', 'hash': md5hex("e" * 25), + 'content_type': 'text/plain', 'bytes': '25'}, + {'name': '/gettest/f_30', 'hash': md5hex("f" * 30), + 'content_type': 'text/plain', 'bytes': '30'}, + {'name': '/gettest/g_35', 'hash': md5hex("g" * 35), + 'content_type': 'text/plain', 'bytes': '35'}, + {'name': '/gettest/h_40', 'hash': md5hex("h" * 40), + 'content_type': 'text/plain', 'bytes': '40'}, + {'name': '/gettest/i_45', 'hash': md5hex("i" * 45), + 'content_type': 'text/plain', 'bytes': '45'}, + {'name': '/gettest/j_50', 'hash': md5hex("j" * 50), + 'content_type': 'text/plain', 'bytes': '50'}, + {'name': '/gettest/k_55', 'hash': md5hex("k" * 55), + 'content_type': 'text/plain', 'bytes': '55'}, + {'name': '/gettest/l_60', 'hash': md5hex("l" * 60), + 'content_type': 'text/plain', 'bytes': '60'}]) + self.app.register( + 'GET', '/v1/AUTH_test/gettest/manifest-abcdefghijkl', + swob.HTTPOk, { + 'Content-Type': 'application/json', + 'X-Static-Large-Object': 'true', + 'Etag': md5(_abcdefghijkl_manifest_json).hexdigest()}, + _abcdefghijkl_manifest_json) + self.manifest_abcd_etag = md5hex( md5hex("a" * 5) + md5hex(md5hex("b" * 10) + md5hex("c" * 15)) + md5hex("d" * 20)) @@ -1361,6 +1404,65 @@ class TestSloGetManifest(SloTestCase): 'bytes=0-14,0-14', 'bytes=0-19,0-19']) + def test_get_manifest_ratelimiting(self): + req = Request.blank( + '/v1/AUTH_test/gettest/manifest-abcdefghijkl', + environ={'REQUEST_METHOD': 'GET'}) + + the_time = [time.time()] + sleeps = [] + + def mock_time(): + return the_time[0] + + def mock_sleep(duration): + sleeps.append(duration) + the_time[0] += duration + + with patch('time.time', mock_time), \ + patch('eventlet.sleep', mock_sleep), \ + patch.object(self.slo, 'rate_limit_under_size', 999999999), \ + patch.object(self.slo, 'rate_limit_after_segment', 0): + status, headers, body = self.call_slo(req) + + self.assertEqual(status, '200 OK') # sanity check + self.assertEqual(sleeps, [2.0, 2.0, 2.0, 2.0, 2.0]) + + # give the client the first 4 segments without ratelimiting; we'll + # sleep less + del sleeps[:] + with patch('time.time', mock_time), \ + patch('eventlet.sleep', mock_sleep), \ + patch.object(self.slo, 'rate_limit_under_size', 999999999), \ + patch.object(self.slo, 'rate_limit_after_segment', 4): + status, headers, body = self.call_slo(req) + + self.assertEqual(status, '200 OK') # sanity check + self.assertEqual(sleeps, [2.0, 2.0, 2.0]) + + # ratelimit segments under 35 bytes; this affects a-f + del sleeps[:] + with patch('time.time', mock_time), \ + patch('eventlet.sleep', mock_sleep), \ + patch.object(self.slo, 'rate_limit_under_size', 35), \ + patch.object(self.slo, 'rate_limit_after_segment', 0): + status, headers, body = self.call_slo(req) + + self.assertEqual(status, '200 OK') # sanity check + self.assertEqual(sleeps, [2.0, 2.0]) + + # ratelimit segments under 36 bytes; this now affects a-g, netting + # us one more sleep than before + del sleeps[:] + with patch('time.time', mock_time), \ + patch('eventlet.sleep', mock_sleep), \ + patch.object(self.slo, 'rate_limit_under_size', 36), \ + patch.object(self.slo, 'rate_limit_after_segment', 0): + status, headers, body = self.call_slo(req) + + self.assertEqual(status, '200 OK') # sanity check + self.assertEqual(sleeps, [2.0, 2.0, 2.0]) + def test_if_none_match_matches(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', @@ -2446,8 +2548,7 @@ class TestSwiftInfo(unittest.TestCase): self.assertTrue('slo' in swift_info) self.assertEqual(swift_info['slo'].get('max_manifest_segments'), mware.max_manifest_segments) - self.assertEqual(swift_info['slo'].get('min_segment_size'), - mware.min_segment_size) + self.assertEqual(swift_info['slo'].get('min_segment_size'), 1) self.assertEqual(swift_info['slo'].get('max_manifest_size'), mware.max_manifest_size) diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 1de31aa438..3923b35f8a 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -3902,6 +3902,26 @@ class TestRateLimitedIterator(unittest.TestCase): # first element. self.assertEqual(len(got), 11) + def test_rate_limiting_sometimes(self): + + def testfunc(): + limited_iterator = utils.RateLimitedIterator( + range(9999), 100, + ratelimit_if=lambda item: item % 23 != 0) + got = [] + started_at = time.time() + try: + while time.time() - started_at < 0.5: + got.append(next(limited_iterator)) + except StopIteration: + pass + return got + + got = self.run_under_pseudo_time(testfunc) + # we'd get 51 without the ratelimit_if, but because 0, 23 and 46 + # weren't subject to ratelimiting, we get 54 instead + self.assertEqual(len(got), 54) + def test_limit_after(self): def testfunc():