openstack/swift
Code Issues Proposed changes

Port FileLikeIter to Python 3

Browse Source 
Port FileLikeIter and _MultipartMimeFileLikeObject and
swift.common.utils to Python 3:

* Add a __next__() alias to the next() method. On Python 3, the
  next() method is no more used, __next__() is required.
* Use literal byte strings: FileLikeIter _MultipartMimeFileLikeObject
  are written to handle binary files.
* test_close(): replace .FileLikeIter('abcdef') with
  FileLikeIter([b'a', b'b', b'c']). On Python 3, list(b'abc') returns
  [97, 98, 99], whereas ['a', 'b', 'c'] is returned on Python 2.
* Update unit FileLikeIter tests to use byte strings.

Change-Id: Ibacddb70b22f624ecd83e374749578feddf8bca8
This commit is contained in:
Victor Stinner 2015-10-19 16:38:24 +02:00 committed by Samuel Merritt
parent b9fd530657
commit 6c32da14f4
2 changed files with 48 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
swift/common/utils.py
View File

@ -453,6 +453,8 @@ class FileLikeIter(object):
def __init__(self, iterable):
"""
Wraps an iterable to behave as a file-like object.
The iterable must yield bytes strings.
"""
self.iterator = iter(iterable)
self.buf = None
@ -473,10 +475,11 @@ class FileLikeIter(object):
return rv
else:
return next(self.iterator)
__next__ = next
def read(self, size=-1):
"""
read([size]) -> read at most size bytes, returned as a string.
read([size]) -> read at most size bytes, returned as a bytes string.
If the size argument is negative or omitted, read until EOF is reached.
Notice that when in non-blocking mode, less data than what was
@ -485,9 +488,9 @@ class FileLikeIter(object):
if self.closed:
raise ValueError('I/O operation on closed file')
if size < 0:
return ''.join(self)
return b''.join(self)
elif not size:
chunk = ''
chunk = b''
elif self.buf:
chunk = self.buf
self.buf = None
@ -495,7 +498,7 @@ class FileLikeIter(object):
try:
chunk = next(self.iterator)
except StopIteration:
return ''
return b''
if len(chunk) > size:
self.buf = chunk[size:]
chunk = chunk[:size]
@ -503,7 +506,7 @@ class FileLikeIter(object):
def readline(self, size=-1):
"""
readline([size]) -> next line from the file, as a string.
readline([size]) -> next line from the file, as a bytes string.
Retain newline. A non-negative size argument limits the maximum
number of bytes to return (an incomplete line may be returned then).
@ -511,8 +514,8 @@ class FileLikeIter(object):
"""
if self.closed:
raise ValueError('I/O operation on closed file')
data = ''
while '\n' not in data and (size < 0 or len(data) < size):
data = b''
while b'\n' not in data and (size < 0 or len(data) < size):
if size < 0:
chunk = self.read(1024)
else:
@ -520,8 +523,8 @@ class FileLikeIter(object):
if not chunk:
break
data += chunk
if '\n' in data:
data, sep, rest = data.partition('\n')
if b'\n' in data:
data, sep, rest = data.partition(b'\n')
data += sep
if self.buf:
self.buf = rest + self.buf
@ -531,7 +534,7 @@ class FileLikeIter(object):
def readlines(self, sizehint=-1):
"""
readlines([size]) -> list of strings, each a line from the file.
readlines([size]) -> list of bytes strings, each a line from the file.
Call readline() repeatedly and return a list of the lines so read.
The optional size argument, if given, is an approximate bound on the
@ -3304,7 +3307,7 @@ class _MultipartMimeFileLikeObject(object):
if not length:
length = self.read_chunk_size
if self.no_more_data_for_this_file:
return ''
return b''
# read enough data to know whether we're going to run
# into a boundary in next [length] bytes
@ -3330,14 +3333,14 @@ class _MultipartMimeFileLikeObject(object):
# if it does, just return data up to the boundary
else:
ret, self.input_buffer = self.input_buffer.split(self.boundary, 1)
self.no_more_files = self.input_buffer.startswith('--')
self.no_more_files = self.input_buffer.startswith(b'--')
self.no_more_data_for_this_file = True
self.input_buffer = self.input_buffer[2:]
return ret
def readline(self):
if self.no_more_data_for_this_file:
return ''
return b''
boundary_pos = newline_pos = -1
while newline_pos < 0 and boundary_pos < 0:
try:
@ -3345,7 +3348,7 @@ class _MultipartMimeFileLikeObject(object):
except (IOError, ValueError) as e:
raise swift.common.exceptions.ChunkReadError(str(e))
self.input_buffer += chunk
newline_pos = self.input_buffer.find('\r\n')
newline_pos = self.input_buffer.find(b'\r\n')
boundary_pos = self.input_buffer.find(self.boundary)
if not chunk:
self.no_more_files = True
@ -3354,7 +3357,7 @@ class _MultipartMimeFileLikeObject(object):
if newline_pos >= 0 and \
(boundary_pos < 0 or newline_pos < boundary_pos):
# Use self.read to ensure any logic there happens...
ret = ''
ret = b''
to_read = newline_pos + 2
while to_read > 0:
chunk = self.read(to_read)

55
test/unit/common/test_utils.py
View File

@ -3466,14 +3466,14 @@ class TestSwiftInfo(unittest.TestCase):
class TestFileLikeIter(unittest.TestCase):
def test_iter_file_iter(self):
in_iter = ['abc', 'de', 'fghijk', 'l']
in_iter = [b'abc', b'de', b'fghijk', b'l']
chunks = []
for chunk in utils.FileLikeIter(in_iter):
chunks.append(chunk)
self.assertEqual(chunks, in_iter)
def test_next(self):
in_iter = ['abc', 'de', 'fghijk', 'l']
in_iter = [b'abc', b'de', b'fghijk', b'l']
chunks = []
iter_file = utils.FileLikeIter(in_iter)
while True:
@ -3485,12 +3485,12 @@ class TestFileLikeIter(unittest.TestCase):
self.assertEqual(chunks, in_iter)
def test_read(self):
in_iter = ['abc', 'de', 'fghijk', 'l']
in_iter = [b'abc', b'de', b'fghijk', b'l']
iter_file = utils.FileLikeIter(in_iter)
self.assertEqual(iter_file.read(), ''.join(in_iter))
self.assertEqual(iter_file.read(), b''.join(in_iter))
def test_read_with_size(self):
in_iter = ['abc', 'de', 'fghijk', 'l']
in_iter = [b'abc', b'de', b'fghijk', b'l']
chunks = []
iter_file = utils.FileLikeIter(in_iter)
while True:
@ -3499,14 +3499,15 @@ class TestFileLikeIter(unittest.TestCase):
break
self.assertTrue(len(chunk) <= 2)
chunks.append(chunk)
self.assertEqual(''.join(chunks), ''.join(in_iter))
self.assertEqual(b''.join(chunks), b''.join(in_iter))
def test_read_with_size_zero(self):
# makes little sense, but file supports it, so...
self.assertEqual(utils.FileLikeIter('abc').read(0), '')
self.assertEqual(utils.FileLikeIter(b'abc').read(0), b'')
def test_readline(self):
in_iter = ['abc\n', 'd', '\nef', 'g\nh', '\nij\n\nk\n', 'trailing.']
in_iter = [b'abc\n', b'd', b'\nef', b'g\nh', b'\nij\n\nk\n',
b'trailing.']
lines = []
iter_file = utils.FileLikeIter(in_iter)
while True:
@ -3516,22 +3517,23 @@ class TestFileLikeIter(unittest.TestCase):
lines.append(line)
self.assertEqual(
lines,
[v if v == 'trailing.' else v + '\n'
for v in ''.join(in_iter).split('\n')])
[v if v == b'trailing.' else v + b'\n'
for v in b''.join(in_iter).split(b'\n')])
def test_readline2(self):
self.assertEqual(
utils.FileLikeIter(['abc', 'def\n']).readline(4),
'abcd')
utils.FileLikeIter([b'abc', b'def\n']).readline(4),
b'abcd')
def test_readline3(self):
self.assertEqual(
utils.FileLikeIter(['a' * 1111, 'bc\ndef']).readline(),
('a' * 1111) + 'bc\n')
utils.FileLikeIter([b'a' * 1111, b'bc\ndef']).readline(),
(b'a' * 1111) + b'bc\n')
def test_readline_with_size(self):
in_iter = ['abc\n', 'd', '\nef', 'g\nh', '\nij\n\nk\n', 'trailing.']
in_iter = [b'abc\n', b'd', b'\nef', b'g\nh', b'\nij\n\nk\n',
b'trailing.']
lines = []
iter_file = utils.FileLikeIter(in_iter)
while True:
@ -3541,19 +3543,21 @@ class TestFileLikeIter(unittest.TestCase):
lines.append(line)
self.assertEqual(
lines,
['ab', 'c\n', 'd\n', 'ef', 'g\n', 'h\n', 'ij', '\n', '\n', 'k\n',
'tr', 'ai', 'li', 'ng', '.'])
[b'ab', b'c\n', b'd\n', b'ef', b'g\n', b'h\n', b'ij', b'\n', b'\n',
b'k\n', b'tr', b'ai', b'li', b'ng', b'.'])
def test_readlines(self):
in_iter = ['abc\n', 'd', '\nef', 'g\nh', '\nij\n\nk\n', 'trailing.']
in_iter = [b'abc\n', b'd', b'\nef', b'g\nh', b'\nij\n\nk\n',
b'trailing.']
lines = utils.FileLikeIter(in_iter).readlines()
self.assertEqual(
lines,
[v if v == 'trailing.' else v + '\n'
for v in ''.join(in_iter).split('\n')])
[v if v == b'trailing.' else v + b'\n'
for v in b''.join(in_iter).split(b'\n')])
def test_readlines_with_size(self):
in_iter = ['abc\n', 'd', '\nef', 'g\nh', '\nij\n\nk\n', 'trailing.']
in_iter = [b'abc\n', b'd', b'\nef', b'g\nh', b'\nij\n\nk\n',
b'trailing.']
iter_file = utils.FileLikeIter(in_iter)
lists_of_lines = []
while True:
@ -3563,12 +3567,13 @@ class TestFileLikeIter(unittest.TestCase):
lists_of_lines.append(lines)
self.assertEqual(
lists_of_lines,
[['ab'], ['c\n'], ['d\n'], ['ef'], ['g\n'], ['h\n'], ['ij'],
['\n', '\n'], ['k\n'], ['tr'], ['ai'], ['li'], ['ng'], ['.']])
[[b'ab'], [b'c\n'], [b'd\n'], [b'ef'], [b'g\n'], [b'h\n'], [b'ij'],
[b'\n', b'\n'], [b'k\n'], [b'tr'], [b'ai'], [b'li'], [b'ng'],
[b'.']])
def test_close(self):
iter_file = utils.FileLikeIter('abcdef')
self.assertEqual(next(iter_file), 'a')
iter_file = utils.FileLikeIter([b'a', b'b', b'c'])
self.assertEqual(next(iter_file), b'a')
iter_file.close()
self.assertTrue(iter_file.closed)
self.assertRaises(ValueError, iter_file.next)