openstack/swift
Code Issues Proposed changes

Port FileLikeIter to Python 3

Browse Source 
Port FileLikeIter and _MultipartMimeFileLikeObject and
swift.common.utils to Python 3:

* Add a __next__() alias to the next() method. On Python 3, the
  next() method is no more used, __next__() is required.
* Use literal byte strings: FileLikeIter _MultipartMimeFileLikeObject
  are written to handle binary files.
* test_close(): replace .FileLikeIter('abcdef') with
  FileLikeIter([b'a', b'b', b'c']). On Python 3, list(b'abc') returns
  [97, 98, 99], whereas ['a', 'b', 'c'] is returned on Python 2.
* Update unit FileLikeIter tests to use byte strings.

Change-Id: Ibacddb70b22f624ecd83e374749578feddf8bca8
This commit is contained in:
Victor Stinner 2015-10-19 16:38:24 +02:00 committed by Samuel Merritt
parent b9fd530657
commit 6c32da14f4
2 changed files with 48 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
swift/common/utils.py
View File

@ -453,6 +453,8 @@ class FileLikeIter(object):
def __init__(self, iterable): def __init__(self, iterable):
""" """
Wraps an iterable to behave as a file-like object. Wraps an iterable to behave as a file-like object.
The iterable must yield bytes strings.
""" """
self.iterator = iter(iterable) self.iterator = iter(iterable)
self.buf = None self.buf = None
@ -473,10 +475,11 @@ class FileLikeIter(object):
return rv return rv
else: else:
return next(self.iterator) return next(self.iterator)
__next__ = next
def read(self, size=-1): def read(self, size=-1):
""" """
read([size]) -> read at most size bytes, returned as a string. read([size]) -> read at most size bytes, returned as a bytes string.
If the size argument is negative or omitted, read until EOF is reached. If the size argument is negative or omitted, read until EOF is reached.
Notice that when in non-blocking mode, less data than what was Notice that when in non-blocking mode, less data than what was
@ -485,9 +488,9 @@ class FileLikeIter(object):
if self.closed: if self.closed:
raise ValueError('I/O operation on closed file') raise ValueError('I/O operation on closed file')
if size < 0: if size < 0:
return ''.join(self) return b''.join(self)
elif not size: elif not size:
chunk = '' chunk = b''
elif self.buf: elif self.buf:
chunk = self.buf chunk = self.buf
self.buf = None self.buf = None
@ -495,7 +498,7 @@ class FileLikeIter(object):
try: try:
chunk = next(self.iterator) chunk = next(self.iterator)
except StopIteration: except StopIteration:
return '' return b''
if len(chunk) > size: if len(chunk) > size:
self.buf = chunk[size:] self.buf = chunk[size:]
chunk = chunk[:size] chunk = chunk[:size]
@ -503,7 +506,7 @@ class FileLikeIter(object):
def readline(self, size=-1): def readline(self, size=-1):
""" """
readline([size]) -> next line from the file, as a string. readline([size]) -> next line from the file, as a bytes string.
Retain newline. A non-negative size argument limits the maximum Retain newline. A non-negative size argument limits the maximum
number of bytes to return (an incomplete line may be returned then). number of bytes to return (an incomplete line may be returned then).
@ -511,8 +514,8 @@ class FileLikeIter(object):
""" """
if self.closed: if self.closed:
raise ValueError('I/O operation on closed file') raise ValueError('I/O operation on closed file')
data = '' data = b''
while '\n' not in data and (size < 0 or len(data) < size): while b'\n' not in data and (size < 0 or len(data) < size):
if size < 0: if size < 0:
chunk = self.read(1024) chunk = self.read(1024)
else: else:
@ -520,8 +523,8 @@ class FileLikeIter(object):
if not chunk: if not chunk:
break break
data += chunk data += chunk
if '\n' in data: if b'\n' in data:
data, sep, rest = data.partition('\n') data, sep, rest = data.partition(b'\n')
data += sep data += sep
if self.buf: if self.buf:
self.buf = rest + self.buf self.buf = rest + self.buf
@ -531,7 +534,7 @@ class FileLikeIter(object):
def readlines(self, sizehint=-1): def readlines(self, sizehint=-1):
""" """
readlines([size]) -> list of strings, each a line from the file. readlines([size]) -> list of bytes strings, each a line from the file.
Call readline() repeatedly and return a list of the lines so read. Call readline() repeatedly and return a list of the lines so read.
The optional size argument, if given, is an approximate bound on the The optional size argument, if given, is an approximate bound on the
@ -3304,7 +3307,7 @@ class _MultipartMimeFileLikeObject(object):
if not length: if not length:
length = self.read_chunk_size length = self.read_chunk_size
if self.no_more_data_for_this_file: if self.no_more_data_for_this_file:
return '' return b''
# read enough data to know whether we're going to run # read enough data to know whether we're going to run
# into a boundary in next [length] bytes # into a boundary in next [length] bytes
@ -3330,14 +3333,14 @@ class _MultipartMimeFileLikeObject(object):
# if it does, just return data up to the boundary # if it does, just return data up to the boundary
else: else:
ret, self.input_buffer = self.input_buffer.split(self.boundary, 1) ret, self.input_buffer = self.input_buffer.split(self.boundary, 1)
self.no_more_files = self.input_buffer.startswith('--') self.no_more_files = self.input_buffer.startswith(b'--')
self.no_more_data_for_this_file = True self.no_more_data_for_this_file = True
self.input_buffer = self.input_buffer[2:] self.input_buffer = self.input_buffer[2:]
return ret return ret
def readline(self): def readline(self):
if self.no_more_data_for_this_file: if self.no_more_data_for_this_file:
return '' return b''
boundary_pos = newline_pos = -1 boundary_pos = newline_pos = -1
while newline_pos < 0 and boundary_pos < 0: while newline_pos < 0 and boundary_pos < 0:
try: try:
@ -3345,7 +3348,7 @@ class _MultipartMimeFileLikeObject(object):
except (IOError, ValueError) as e: except (IOError, ValueError) as e:
raise swift.common.exceptions.ChunkReadError(str(e)) raise swift.common.exceptions.ChunkReadError(str(e))
self.input_buffer += chunk self.input_buffer += chunk
newline_pos = self.input_buffer.find('\r\n') newline_pos = self.input_buffer.find(b'\r\n')
boundary_pos = self.input_buffer.find(self.boundary) boundary_pos = self.input_buffer.find(self.boundary)
if not chunk: if not chunk:
self.no_more_files = True self.no_more_files = True
@ -3354,7 +3357,7 @@ class _MultipartMimeFileLikeObject(object):
if newline_pos >= 0 and \ if newline_pos >= 0 and \
(boundary_pos < 0 or newline_pos < boundary_pos): (boundary_pos < 0 or newline_pos < boundary_pos):
# Use self.read to ensure any logic there happens... # Use self.read to ensure any logic there happens...
ret = '' ret = b''
to_read = newline_pos + 2 to_read = newline_pos + 2
while to_read > 0: while to_read > 0:
chunk = self.read(to_read) chunk = self.read(to_read)

55
test/unit/common/test_utils.py
View File

@ -3466,14 +3466,14 @@ class TestSwiftInfo(unittest.TestCase):
class TestFileLikeIter(unittest.TestCase): class TestFileLikeIter(unittest.TestCase):
def test_iter_file_iter(self): def test_iter_file_iter(self):
in_iter = ['abc', 'de', 'fghijk', 'l'] in_iter = [b'abc', b'de', b'fghijk', b'l']
chunks = [] chunks = []
for chunk in utils.FileLikeIter(in_iter): for chunk in utils.FileLikeIter(in_iter):
chunks.append(chunk) chunks.append(chunk)
self.assertEqual(chunks, in_iter) self.assertEqual(chunks, in_iter)
def test_next(self): def test_next(self):
in_iter = ['abc', 'de', 'fghijk', 'l'] in_iter = [b'abc', b'de', b'fghijk', b'l']
chunks = [] chunks = []
iter_file = utils.FileLikeIter(in_iter) iter_file = utils.FileLikeIter(in_iter)
while True: while True:
@ -3485,12 +3485,12 @@ class TestFileLikeIter(unittest.TestCase):
self.assertEqual(chunks, in_iter) self.assertEqual(chunks, in_iter)
def test_read(self): def test_read(self):
in_iter = ['abc', 'de', 'fghijk', 'l'] in_iter = [b'abc', b'de', b'fghijk', b'l']
iter_file = utils.FileLikeIter(in_iter) iter_file = utils.FileLikeIter(in_iter)
self.assertEqual(iter_file.read(), ''.join(in_iter)) self.assertEqual(iter_file.read(), b''.join(in_iter))
def test_read_with_size(self): def test_read_with_size(self):
in_iter = ['abc', 'de', 'fghijk', 'l'] in_iter = [b'abc', b'de', b'fghijk', b'l']
chunks = [] chunks = []
iter_file = utils.FileLikeIter(in_iter) iter_file = utils.FileLikeIter(in_iter)
while True: while True:
@ -3499,14 +3499,15 @@ class TestFileLikeIter(unittest.TestCase):
break break
self.assertTrue(len(chunk) <= 2) self.assertTrue(len(chunk) <= 2)
chunks.append(chunk) chunks.append(chunk)
self.assertEqual(''.join(chunks), ''.join(in_iter)) self.assertEqual(b''.join(chunks), b''.join(in_iter))
def test_read_with_size_zero(self): def test_read_with_size_zero(self):
# makes little sense, but file supports it, so... # makes little sense, but file supports it, so...
self.assertEqual(utils.FileLikeIter('abc').read(0), '') self.assertEqual(utils.FileLikeIter(b'abc').read(0), b'')
def test_readline(self): def test_readline(self):
in_iter = ['abc\n', 'd', '\nef', 'g\nh', '\nij\n\nk\n', 'trailing.'] in_iter = [b'abc\n', b'd', b'\nef', b'g\nh', b'\nij\n\nk\n',
b'trailing.']
lines = [] lines = []
iter_file = utils.FileLikeIter(in_iter) iter_file = utils.FileLikeIter(in_iter)
while True: while True:
@ -3516,22 +3517,23 @@ class TestFileLikeIter(unittest.TestCase):
lines.append(line) lines.append(line)
self.assertEqual( self.assertEqual(
lines, lines,
[v if v == 'trailing.' else v + '\n' [v if v == b'trailing.' else v + b'\n'
for v in ''.join(in_iter).split('\n')]) for v in b''.join(in_iter).split(b'\n')])
def test_readline2(self): def test_readline2(self):
self.assertEqual( self.assertEqual(
utils.FileLikeIter(['abc', 'def\n']).readline(4), utils.FileLikeIter([b'abc', b'def\n']).readline(4),
'abcd') b'abcd')
def test_readline3(self): def test_readline3(self):
self.assertEqual( self.assertEqual(
utils.FileLikeIter(['a' * 1111, 'bc\ndef']).readline(), utils.FileLikeIter([b'a' * 1111, b'bc\ndef']).readline(),
('a' * 1111) + 'bc\n') (b'a' * 1111) + b'bc\n')
def test_readline_with_size(self): def test_readline_with_size(self):
in_iter = ['abc\n', 'd', '\nef', 'g\nh', '\nij\n\nk\n', 'trailing.'] in_iter = [b'abc\n', b'd', b'\nef', b'g\nh', b'\nij\n\nk\n',
b'trailing.']
lines = [] lines = []
iter_file = utils.FileLikeIter(in_iter) iter_file = utils.FileLikeIter(in_iter)
while True: while True:
@ -3541,19 +3543,21 @@ class TestFileLikeIter(unittest.TestCase):
lines.append(line) lines.append(line)
self.assertEqual( self.assertEqual(
lines, lines,
['ab', 'c\n', 'd\n', 'ef', 'g\n', 'h\n', 'ij', '\n', '\n', 'k\n', [b'ab', b'c\n', b'd\n', b'ef', b'g\n', b'h\n', b'ij', b'\n', b'\n',
'tr', 'ai', 'li', 'ng', '.']) b'k\n', b'tr', b'ai', b'li', b'ng', b'.'])
def test_readlines(self): def test_readlines(self):
in_iter = ['abc\n', 'd', '\nef', 'g\nh', '\nij\n\nk\n', 'trailing.'] in_iter = [b'abc\n', b'd', b'\nef', b'g\nh', b'\nij\n\nk\n',
b'trailing.']
lines = utils.FileLikeIter(in_iter).readlines() lines = utils.FileLikeIter(in_iter).readlines()
self.assertEqual( self.assertEqual(
lines, lines,
[v if v == 'trailing.' else v + '\n' [v if v == b'trailing.' else v + b'\n'
for v in ''.join(in_iter).split('\n')]) for v in b''.join(in_iter).split(b'\n')])
def test_readlines_with_size(self): def test_readlines_with_size(self):
in_iter = ['abc\n', 'd', '\nef', 'g\nh', '\nij\n\nk\n', 'trailing.'] in_iter = [b'abc\n', b'd', b'\nef', b'g\nh', b'\nij\n\nk\n',
b'trailing.']
iter_file = utils.FileLikeIter(in_iter) iter_file = utils.FileLikeIter(in_iter)
lists_of_lines = [] lists_of_lines = []
while True: while True:
@ -3563,12 +3567,13 @@ class TestFileLikeIter(unittest.TestCase):
lists_of_lines.append(lines) lists_of_lines.append(lines)
self.assertEqual( self.assertEqual(
lists_of_lines, lists_of_lines,
[['ab'], ['c\n'], ['d\n'], ['ef'], ['g\n'], ['h\n'], ['ij'], [[b'ab'], [b'c\n'], [b'd\n'], [b'ef'], [b'g\n'], [b'h\n'], [b'ij'],
['\n', '\n'], ['k\n'], ['tr'], ['ai'], ['li'], ['ng'], ['.']]) [b'\n', b'\n'], [b'k\n'], [b'tr'], [b'ai'], [b'li'], [b'ng'],
[b'.']])
def test_close(self): def test_close(self):
iter_file = utils.FileLikeIter('abcdef') iter_file = utils.FileLikeIter([b'a', b'b', b'c'])
self.assertEqual(next(iter_file), 'a') self.assertEqual(next(iter_file), b'a')
iter_file.close() iter_file.close()
self.assertTrue(iter_file.closed) self.assertTrue(iter_file.closed)
self.assertRaises(ValueError, iter_file.next) self.assertRaises(ValueError, iter_file.next)