openstack/swift
Code Issues Proposed changes

ECFragGetter: assume policy.fragment_size is non-zero

Browse Source 
Simplify ECFragGetter by removing code that guards against the policy
fragment_size being None or zero.

Policy fragment_size must be > 0: the fragment_size is based on the
ec_segment_size, which is verified as > 0 when constructing an EC
policy. This is asserted by test_parse_storage_policies in
test.unit.common.test_storage_policy.TestStoragePolicies.

Also, rename client_chunk_size to fragment_size for clarity.

Change-Id: Ie1efaab3bd0510275d534b5c023cb73c98bec90d
This commit is contained in:
Alistair Coles 2023-04-26 11:20:42 +01:00
parent 0f95870c51
commit 50c4ea032d
2 changed files with 10 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
swift/proxy/controllers/obj.py
View File

@ -2497,7 +2497,7 @@ class ECFragGetter(object):
self.backend_headers = backend_headers
self.header_provider = header_provider
self.req_query_string = req.query_string
self.client_chunk_size = policy.fragment_size
self.fragment_size = policy.fragment_size
self.skip_bytes = 0
self.bytes_used_from_backend = 0
self.source = self.node = None
@ -2578,8 +2578,8 @@ class ECFragGetter(object):
def learn_size_from_content_range(self, start, end, length):
"""
If client_chunk_size is set, makes sure we yield things starting on
chunk boundaries based on the Content-Range header in the response.
Make sure we yield things starting on fragment boundaries based on the
Content-Range header in the response.
Sets our Range header's first byterange to the value learned from
the Content-Range header in the response; if we were given a
@ -2593,8 +2593,7 @@ class ECFragGetter(object):
if length == 0:
return
if self.client_chunk_size:
self.skip_bytes = bytes_to_skip(self.client_chunk_size, start)
self.skip_bytes = bytes_to_skip(self.fragment_size, start)
if 'Range' in self.backend_headers:
try:
@ -2725,10 +2724,9 @@ class ECFragGetter(object):
self.bytes_used_from_backend += len(buf)
buf = b''
client_chunk_size = self.client_chunk_size or len(buf)
while buf and (len(buf) >= client_chunk_size or not chunk):
client_chunk = buf[:client_chunk_size]
buf = buf[client_chunk_size:]
while buf and (len(buf) >= self.fragment_size or not chunk):
client_chunk = buf[:self.fragment_size]
buf = buf[self.fragment_size:]
with WatchdogTimeout(self.app.watchdog,
self.app.client_timeout,
ChunkWriteTimeout):

21
test/unit/proxy/controllers/test_obj.py
View File

@ -6715,27 +6715,12 @@ class TestECFragGetter(BaseObjectControllerMixin, unittest.TestCase):
b''.join(it)
self.assertEqual('9 seconds', str(cm.exception))
def test_iter_bytes_from_response_part_null_chunk_size(self):
# we don't expect a policy to have fragment_size None or zero but
# verify that the getter is defensive
self.getter.client_chunk_size = None
part = FileLikeIter([b'some', b'thing', b''])
it = self.getter.iter_bytes_from_response_part(part, nbytes=None)
self.assertEqual(b'something', b''.join(it))
self.getter.client_chunk_size = 0
part = FileLikeIter([b'some', b'thing', b''])
it = self.getter.iter_bytes_from_response_part(part, nbytes=None)
self.assertEqual(b'something', b''.join(it))
def test_iter_bytes_from_response_part_small_chunk_size(self):
# we don't expect a policy to have fragment_size None or zero but
# verify that the getter is defensive
self.getter.client_chunk_size = 4
def test_iter_bytes_from_response_part_small_fragment_size(self):
self.getter.fragment_size = 4
part = FileLikeIter([b'some', b'thing', b''])
it = self.getter.iter_bytes_from_response_part(part, nbytes=None)
self.assertEqual([b'some', b'thin', b'g'], [ch for ch in it])
self.getter.client_chunk_size = 1
self.getter.fragment_size = 1
part = FileLikeIter([b'some', b'thing', b''])
it = self.getter.iter_bytes_from_response_part(part, nbytes=None)
self.assertEqual([c.encode() for c in 'something'], [ch for ch in it])