From 6c32da14f48b435aa4e5efea48904bd288a532a2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 19 Oct 2015 16:38:24 +0200 Subject: [PATCH] Port FileLikeIter to Python 3 Port FileLikeIter and _MultipartMimeFileLikeObject and swift.common.utils to Python 3: * Add a __next__() alias to the next() method. On Python 3, the next() method is no more used, __next__() is required. * Use literal byte strings: FileLikeIter _MultipartMimeFileLikeObject are written to handle binary files. * test_close(): replace .FileLikeIter('abcdef') with FileLikeIter([b'a', b'b', b'c']). On Python 3, list(b'abc') returns [97, 98, 99], whereas ['a', 'b', 'c'] is returned on Python 2. * Update unit FileLikeIter tests to use byte strings. Change-Id: Ibacddb70b22f624ecd83e374749578feddf8bca8 --- swift/common/utils.py | 33 ++++++++++---------- test/unit/common/test_utils.py | 55 ++++++++++++++++++---------------- 2 files changed, 48 insertions(+), 40 deletions(-) diff --git a/swift/common/utils.py b/swift/common/utils.py index a7615220a2..e6a69ad272 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -453,6 +453,8 @@ class FileLikeIter(object): def __init__(self, iterable): """ Wraps an iterable to behave as a file-like object. + + The iterable must yield bytes strings. """ self.iterator = iter(iterable) self.buf = None @@ -473,10 +475,11 @@ class FileLikeIter(object): return rv else: return next(self.iterator) + __next__ = next def read(self, size=-1): """ - read([size]) -> read at most size bytes, returned as a string. + read([size]) -> read at most size bytes, returned as a bytes string. If the size argument is negative or omitted, read until EOF is reached. Notice that when in non-blocking mode, less data than what was @@ -485,9 +488,9 @@ class FileLikeIter(object): if self.closed: raise ValueError('I/O operation on closed file') if size < 0: - return ''.join(self) + return b''.join(self) elif not size: - chunk = '' + chunk = b'' elif self.buf: chunk = self.buf self.buf = None @@ -495,7 +498,7 @@ class FileLikeIter(object): try: chunk = next(self.iterator) except StopIteration: - return '' + return b'' if len(chunk) > size: self.buf = chunk[size:] chunk = chunk[:size] @@ -503,7 +506,7 @@ class FileLikeIter(object): def readline(self, size=-1): """ - readline([size]) -> next line from the file, as a string. + readline([size]) -> next line from the file, as a bytes string. Retain newline. A non-negative size argument limits the maximum number of bytes to return (an incomplete line may be returned then). @@ -511,8 +514,8 @@ class FileLikeIter(object): """ if self.closed: raise ValueError('I/O operation on closed file') - data = '' - while '\n' not in data and (size < 0 or len(data) < size): + data = b'' + while b'\n' not in data and (size < 0 or len(data) < size): if size < 0: chunk = self.read(1024) else: @@ -520,8 +523,8 @@ class FileLikeIter(object): if not chunk: break data += chunk - if '\n' in data: - data, sep, rest = data.partition('\n') + if b'\n' in data: + data, sep, rest = data.partition(b'\n') data += sep if self.buf: self.buf = rest + self.buf @@ -531,7 +534,7 @@ class FileLikeIter(object): def readlines(self, sizehint=-1): """ - readlines([size]) -> list of strings, each a line from the file. + readlines([size]) -> list of bytes strings, each a line from the file. Call readline() repeatedly and return a list of the lines so read. The optional size argument, if given, is an approximate bound on the @@ -3304,7 +3307,7 @@ class _MultipartMimeFileLikeObject(object): if not length: length = self.read_chunk_size if self.no_more_data_for_this_file: - return '' + return b'' # read enough data to know whether we're going to run # into a boundary in next [length] bytes @@ -3330,14 +3333,14 @@ class _MultipartMimeFileLikeObject(object): # if it does, just return data up to the boundary else: ret, self.input_buffer = self.input_buffer.split(self.boundary, 1) - self.no_more_files = self.input_buffer.startswith('--') + self.no_more_files = self.input_buffer.startswith(b'--') self.no_more_data_for_this_file = True self.input_buffer = self.input_buffer[2:] return ret def readline(self): if self.no_more_data_for_this_file: - return '' + return b'' boundary_pos = newline_pos = -1 while newline_pos < 0 and boundary_pos < 0: try: @@ -3345,7 +3348,7 @@ class _MultipartMimeFileLikeObject(object): except (IOError, ValueError) as e: raise swift.common.exceptions.ChunkReadError(str(e)) self.input_buffer += chunk - newline_pos = self.input_buffer.find('\r\n') + newline_pos = self.input_buffer.find(b'\r\n') boundary_pos = self.input_buffer.find(self.boundary) if not chunk: self.no_more_files = True @@ -3354,7 +3357,7 @@ class _MultipartMimeFileLikeObject(object): if newline_pos >= 0 and \ (boundary_pos < 0 or newline_pos < boundary_pos): # Use self.read to ensure any logic there happens... - ret = '' + ret = b'' to_read = newline_pos + 2 while to_read > 0: chunk = self.read(to_read) diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 8f5f82eff0..f385078464 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -3466,14 +3466,14 @@ class TestSwiftInfo(unittest.TestCase): class TestFileLikeIter(unittest.TestCase): def test_iter_file_iter(self): - in_iter = ['abc', 'de', 'fghijk', 'l'] + in_iter = [b'abc', b'de', b'fghijk', b'l'] chunks = [] for chunk in utils.FileLikeIter(in_iter): chunks.append(chunk) self.assertEqual(chunks, in_iter) def test_next(self): - in_iter = ['abc', 'de', 'fghijk', 'l'] + in_iter = [b'abc', b'de', b'fghijk', b'l'] chunks = [] iter_file = utils.FileLikeIter(in_iter) while True: @@ -3485,12 +3485,12 @@ class TestFileLikeIter(unittest.TestCase): self.assertEqual(chunks, in_iter) def test_read(self): - in_iter = ['abc', 'de', 'fghijk', 'l'] + in_iter = [b'abc', b'de', b'fghijk', b'l'] iter_file = utils.FileLikeIter(in_iter) - self.assertEqual(iter_file.read(), ''.join(in_iter)) + self.assertEqual(iter_file.read(), b''.join(in_iter)) def test_read_with_size(self): - in_iter = ['abc', 'de', 'fghijk', 'l'] + in_iter = [b'abc', b'de', b'fghijk', b'l'] chunks = [] iter_file = utils.FileLikeIter(in_iter) while True: @@ -3499,14 +3499,15 @@ class TestFileLikeIter(unittest.TestCase): break self.assertTrue(len(chunk) <= 2) chunks.append(chunk) - self.assertEqual(''.join(chunks), ''.join(in_iter)) + self.assertEqual(b''.join(chunks), b''.join(in_iter)) def test_read_with_size_zero(self): # makes little sense, but file supports it, so... - self.assertEqual(utils.FileLikeIter('abc').read(0), '') + self.assertEqual(utils.FileLikeIter(b'abc').read(0), b'') def test_readline(self): - in_iter = ['abc\n', 'd', '\nef', 'g\nh', '\nij\n\nk\n', 'trailing.'] + in_iter = [b'abc\n', b'd', b'\nef', b'g\nh', b'\nij\n\nk\n', + b'trailing.'] lines = [] iter_file = utils.FileLikeIter(in_iter) while True: @@ -3516,22 +3517,23 @@ class TestFileLikeIter(unittest.TestCase): lines.append(line) self.assertEqual( lines, - [v if v == 'trailing.' else v + '\n' - for v in ''.join(in_iter).split('\n')]) + [v if v == b'trailing.' else v + b'\n' + for v in b''.join(in_iter).split(b'\n')]) def test_readline2(self): self.assertEqual( - utils.FileLikeIter(['abc', 'def\n']).readline(4), - 'abcd') + utils.FileLikeIter([b'abc', b'def\n']).readline(4), + b'abcd') def test_readline3(self): self.assertEqual( - utils.FileLikeIter(['a' * 1111, 'bc\ndef']).readline(), - ('a' * 1111) + 'bc\n') + utils.FileLikeIter([b'a' * 1111, b'bc\ndef']).readline(), + (b'a' * 1111) + b'bc\n') def test_readline_with_size(self): - in_iter = ['abc\n', 'd', '\nef', 'g\nh', '\nij\n\nk\n', 'trailing.'] + in_iter = [b'abc\n', b'd', b'\nef', b'g\nh', b'\nij\n\nk\n', + b'trailing.'] lines = [] iter_file = utils.FileLikeIter(in_iter) while True: @@ -3541,19 +3543,21 @@ class TestFileLikeIter(unittest.TestCase): lines.append(line) self.assertEqual( lines, - ['ab', 'c\n', 'd\n', 'ef', 'g\n', 'h\n', 'ij', '\n', '\n', 'k\n', - 'tr', 'ai', 'li', 'ng', '.']) + [b'ab', b'c\n', b'd\n', b'ef', b'g\n', b'h\n', b'ij', b'\n', b'\n', + b'k\n', b'tr', b'ai', b'li', b'ng', b'.']) def test_readlines(self): - in_iter = ['abc\n', 'd', '\nef', 'g\nh', '\nij\n\nk\n', 'trailing.'] + in_iter = [b'abc\n', b'd', b'\nef', b'g\nh', b'\nij\n\nk\n', + b'trailing.'] lines = utils.FileLikeIter(in_iter).readlines() self.assertEqual( lines, - [v if v == 'trailing.' else v + '\n' - for v in ''.join(in_iter).split('\n')]) + [v if v == b'trailing.' else v + b'\n' + for v in b''.join(in_iter).split(b'\n')]) def test_readlines_with_size(self): - in_iter = ['abc\n', 'd', '\nef', 'g\nh', '\nij\n\nk\n', 'trailing.'] + in_iter = [b'abc\n', b'd', b'\nef', b'g\nh', b'\nij\n\nk\n', + b'trailing.'] iter_file = utils.FileLikeIter(in_iter) lists_of_lines = [] while True: @@ -3563,12 +3567,13 @@ class TestFileLikeIter(unittest.TestCase): lists_of_lines.append(lines) self.assertEqual( lists_of_lines, - [['ab'], ['c\n'], ['d\n'], ['ef'], ['g\n'], ['h\n'], ['ij'], - ['\n', '\n'], ['k\n'], ['tr'], ['ai'], ['li'], ['ng'], ['.']]) + [[b'ab'], [b'c\n'], [b'd\n'], [b'ef'], [b'g\n'], [b'h\n'], [b'ij'], + [b'\n', b'\n'], [b'k\n'], [b'tr'], [b'ai'], [b'li'], [b'ng'], + [b'.']]) def test_close(self): - iter_file = utils.FileLikeIter('abcdef') - self.assertEqual(next(iter_file), 'a') + iter_file = utils.FileLikeIter([b'a', b'b', b'c']) + self.assertEqual(next(iter_file), b'a') iter_file.close() self.assertTrue(iter_file.closed) self.assertRaises(ValueError, iter_file.next)