openstack/swift
Code Issues Proposed changes

Merge "Port FileLikeIter to Python 3"

Browse Source
This commit is contained in:
Jenkins 2016-02-10 07:36:50 +00:00 committed by Gerrit Code Review
commit 4e370e5116
2 changed files with 48 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
swift/common/utils.py
View File

@ -454,6 +454,8 @@ class FileLikeIter(object):
def __init__(self, iterable):
"""
Wraps an iterable to behave as a file-like object.
The iterable must yield bytes strings.
"""
self.iterator = iter(iterable)
self.buf = None
@ -474,10 +476,11 @@ class FileLikeIter(object):
return rv
else:
return next(self.iterator)
__next__ = next
def read(self, size=-1):
"""
read([size]) -> read at most size bytes, returned as a string.
read([size]) -> read at most size bytes, returned as a bytes string.
If the size argument is negative or omitted, read until EOF is reached.
Notice that when in non-blocking mode, less data than what was
@ -486,9 +489,9 @@ class FileLikeIter(object):
if self.closed:
raise ValueError('I/O operation on closed file')
if size < 0:
return ''.join(self)
return b''.join(self)
elif not size:
chunk = ''
chunk = b''
elif self.buf:
chunk = self.buf
self.buf = None
@ -496,7 +499,7 @@ class FileLikeIter(object):
try:
chunk = next(self.iterator)
except StopIteration:
return ''
return b''
if len(chunk) > size:
self.buf = chunk[size:]
chunk = chunk[:size]
@ -504,7 +507,7 @@ class FileLikeIter(object):
def readline(self, size=-1):
"""
readline([size]) -> next line from the file, as a string.
readline([size]) -> next line from the file, as a bytes string.
Retain newline. A non-negative size argument limits the maximum
number of bytes to return (an incomplete line may be returned then).
@ -512,8 +515,8 @@ class FileLikeIter(object):
"""
if self.closed:
raise ValueError('I/O operation on closed file')
data = ''
while '\n' not in data and (size < 0 or len(data) < size):
data = b''
while b'\n' not in data and (size < 0 or len(data) < size):
if size < 0:
chunk = self.read(1024)
else:
@ -521,8 +524,8 @@ class FileLikeIter(object):
if not chunk:
break
data += chunk
if '\n' in data:
data, sep, rest = data.partition('\n')
if b'\n' in data:
data, sep, rest = data.partition(b'\n')
data += sep
if self.buf:
self.buf = rest + self.buf
@ -532,7 +535,7 @@ class FileLikeIter(object):
def readlines(self, sizehint=-1):
"""
readlines([size]) -> list of strings, each a line from the file.
readlines([size]) -> list of bytes strings, each a line from the file.
Call readline() repeatedly and return a list of the lines so read.
The optional size argument, if given, is an approximate bound on the
@ -3370,7 +3373,7 @@ class _MultipartMimeFileLikeObject(object):
if not length:
length = self.read_chunk_size
if self.no_more_data_for_this_file:
return ''
return b''
# read enough data to know whether we're going to run
# into a boundary in next [length] bytes
@ -3396,14 +3399,14 @@ class _MultipartMimeFileLikeObject(object):
# if it does, just return data up to the boundary
else:
ret, self.input_buffer = self.input_buffer.split(self.boundary, 1)
self.no_more_files = self.input_buffer.startswith('--')
self.no_more_files = self.input_buffer.startswith(b'--')
self.no_more_data_for_this_file = True
self.input_buffer = self.input_buffer[2:]
return ret
def readline(self):
if self.no_more_data_for_this_file:
return ''
return b''
boundary_pos = newline_pos = -1
while newline_pos < 0 and boundary_pos < 0:
try:
@ -3411,7 +3414,7 @@ class _MultipartMimeFileLikeObject(object):
except (IOError, ValueError) as e:
raise swift.common.exceptions.ChunkReadError(str(e))
self.input_buffer += chunk
newline_pos = self.input_buffer.find('\r\n')
newline_pos = self.input_buffer.find(b'\r\n')
boundary_pos = self.input_buffer.find(self.boundary)
if not chunk:
self.no_more_files = True
@ -3420,7 +3423,7 @@ class _MultipartMimeFileLikeObject(object):
if newline_pos >= 0 and \
(boundary_pos < 0 or newline_pos < boundary_pos):
# Use self.read to ensure any logic there happens...
ret = ''
ret = b''
to_read = newline_pos + 2
while to_read > 0:
chunk = self.read(to_read)

55
test/unit/common/test_utils.py
View File

@ -3493,14 +3493,14 @@ class TestSwiftInfo(unittest.TestCase):
class TestFileLikeIter(unittest.TestCase):
def test_iter_file_iter(self):
in_iter = ['abc', 'de', 'fghijk', 'l']
in_iter = [b'abc', b'de', b'fghijk', b'l']
chunks = []
for chunk in utils.FileLikeIter(in_iter):
chunks.append(chunk)
self.assertEqual(chunks, in_iter)
def test_next(self):
in_iter = ['abc', 'de', 'fghijk', 'l']
in_iter = [b'abc', b'de', b'fghijk', b'l']
chunks = []
iter_file = utils.FileLikeIter(in_iter)
while True:
@ -3512,12 +3512,12 @@ class TestFileLikeIter(unittest.TestCase):
self.assertEqual(chunks, in_iter)
def test_read(self):
in_iter = ['abc', 'de', 'fghijk', 'l']
in_iter = [b'abc', b'de', b'fghijk', b'l']
iter_file = utils.FileLikeIter(in_iter)
self.assertEqual(iter_file.read(), ''.join(in_iter))
self.assertEqual(iter_file.read(), b''.join(in_iter))
def test_read_with_size(self):
in_iter = ['abc', 'de', 'fghijk', 'l']
in_iter = [b'abc', b'de', b'fghijk', b'l']
chunks = []
iter_file = utils.FileLikeIter(in_iter)
while True:
@ -3526,14 +3526,15 @@ class TestFileLikeIter(unittest.TestCase):
break
self.assertTrue(len(chunk) <= 2)
chunks.append(chunk)
self.assertEqual(''.join(chunks), ''.join(in_iter))
self.assertEqual(b''.join(chunks), b''.join(in_iter))
def test_read_with_size_zero(self):
# makes little sense, but file supports it, so...
self.assertEqual(utils.FileLikeIter('abc').read(0), '')
self.assertEqual(utils.FileLikeIter(b'abc').read(0), b'')
def test_readline(self):
in_iter = ['abc\n', 'd', '\nef', 'g\nh', '\nij\n\nk\n', 'trailing.']
in_iter = [b'abc\n', b'd', b'\nef', b'g\nh', b'\nij\n\nk\n',
b'trailing.']
lines = []
iter_file = utils.FileLikeIter(in_iter)
while True:
@ -3543,22 +3544,23 @@ class TestFileLikeIter(unittest.TestCase):
lines.append(line)
self.assertEqual(
lines,
[v if v == 'trailing.' else v + '\n'
for v in ''.join(in_iter).split('\n')])
[v if v == b'trailing.' else v + b'\n'
for v in b''.join(in_iter).split(b'\n')])
def test_readline2(self):
self.assertEqual(
utils.FileLikeIter(['abc', 'def\n']).readline(4),
'abcd')
utils.FileLikeIter([b'abc', b'def\n']).readline(4),
b'abcd')
def test_readline3(self):
self.assertEqual(
utils.FileLikeIter(['a' * 1111, 'bc\ndef']).readline(),
('a' * 1111) + 'bc\n')
utils.FileLikeIter([b'a' * 1111, b'bc\ndef']).readline(),
(b'a' * 1111) + b'bc\n')
def test_readline_with_size(self):
in_iter = ['abc\n', 'd', '\nef', 'g\nh', '\nij\n\nk\n', 'trailing.']
in_iter = [b'abc\n', b'd', b'\nef', b'g\nh', b'\nij\n\nk\n',
b'trailing.']
lines = []
iter_file = utils.FileLikeIter(in_iter)
while True:
@ -3568,19 +3570,21 @@ class TestFileLikeIter(unittest.TestCase):
lines.append(line)
self.assertEqual(
lines,
['ab', 'c\n', 'd\n', 'ef', 'g\n', 'h\n', 'ij', '\n', '\n', 'k\n',
'tr', 'ai', 'li', 'ng', '.'])
[b'ab', b'c\n', b'd\n', b'ef', b'g\n', b'h\n', b'ij', b'\n', b'\n',
b'k\n', b'tr', b'ai', b'li', b'ng', b'.'])
def test_readlines(self):
in_iter = ['abc\n', 'd', '\nef', 'g\nh', '\nij\n\nk\n', 'trailing.']
in_iter = [b'abc\n', b'd', b'\nef', b'g\nh', b'\nij\n\nk\n',
b'trailing.']
lines = utils.FileLikeIter(in_iter).readlines()
self.assertEqual(
lines,
[v if v == 'trailing.' else v + '\n'
for v in ''.join(in_iter).split('\n')])
[v if v == b'trailing.' else v + b'\n'
for v in b''.join(in_iter).split(b'\n')])
def test_readlines_with_size(self):
in_iter = ['abc\n', 'd', '\nef', 'g\nh', '\nij\n\nk\n', 'trailing.']
in_iter = [b'abc\n', b'd', b'\nef', b'g\nh', b'\nij\n\nk\n',
b'trailing.']
iter_file = utils.FileLikeIter(in_iter)
lists_of_lines = []
while True:
@ -3590,12 +3594,13 @@ class TestFileLikeIter(unittest.TestCase):
lists_of_lines.append(lines)
self.assertEqual(
lists_of_lines,
[['ab'], ['c\n'], ['d\n'], ['ef'], ['g\n'], ['h\n'], ['ij'],
['\n', '\n'], ['k\n'], ['tr'], ['ai'], ['li'], ['ng'], ['.']])
[[b'ab'], [b'c\n'], [b'd\n'], [b'ef'], [b'g\n'], [b'h\n'], [b'ij'],
[b'\n', b'\n'], [b'k\n'], [b'tr'], [b'ai'], [b'li'], [b'ng'],
[b'.']])
def test_close(self):
iter_file = utils.FileLikeIter('abcdef')
self.assertEqual(next(iter_file), 'a')
iter_file = utils.FileLikeIter([b'a', b'b', b'c'])
self.assertEqual(next(iter_file), b'a')
iter_file.close()
self.assertTrue(iter_file.closed)
self.assertRaises(ValueError, iter_file.next)