Added utils.FileLikeIter
A really simple version of this was in container sync already, and I needed a more complete version for work I'm doing, and I noticed https://review.openstack.org/#/c/33405/ was also making use of it. So, here's a more full version. If https://review.openstack.org/#/c/33405/ lands before this, I'll update it accordingly. Change-Id: Iba66b6a97f65e312e04fdba273e8f4ad1d3e1594
This commit is contained in:
gholt
parent
60c1bc545e
commit
0a77f04893
@ -199,6 +199,122 @@ def get_trans_id_time(trans_id):
|
||||
return None
|
||||
|
||||
|
||||
class FileLikeIter(object):
|
||||
|
||||
def __init__(self, iterable):
|
||||
"""
|
||||
Wraps an iterable to behave as a file-like object.
|
||||
"""
|
||||
self.iterator = iter(iterable)
|
||||
self.buf = None
|
||||
self.closed = False
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def next(self):
|
||||
"""
|
||||
x.next() -> the next value, or raise StopIteration
|
||||
"""
|
||||
if self.closed:
|
||||
raise ValueError('I/O operation on closed file')
|
||||
if self.buf:
|
||||
rv = self.buf
|
||||
self.buf = None
|
||||
return rv
|
||||
else:
|
||||
return self.iterator.next()
|
||||
|
||||
def read(self, size=-1):
|
||||
"""
|
||||
read([size]) -> read at most size bytes, returned as a string.
|
||||
|
||||
If the size argument is negative or omitted, read until EOF is reached.
|
||||
Notice that when in non-blocking mode, less data than what was
|
||||
requested may be returned, even if no size parameter was given.
|
||||
"""
|
||||
if self.closed:
|
||||
raise ValueError('I/O operation on closed file')
|
||||
if size < 0:
|
||||
return ''.join(self)
|
||||
elif not size:
|
||||
chunk = ''
|
||||
elif self.buf:
|
||||
chunk = self.buf
|
||||
self.buf = None
|
||||
else:
|
||||
try:
|
||||
chunk = self.iterator.next()
|
||||
except StopIteration:
|
||||
return ''
|
||||
if len(chunk) > size:
|
||||
self.buf = chunk[size:]
|
||||
chunk = chunk[:size]
|
||||
return chunk
|
||||
|
||||
def readline(self, size=-1):
|
||||
"""
|
||||
readline([size]) -> next line from the file, as a string.
|
||||
|
||||
Retain newline. A non-negative size argument limits the maximum
|
||||
number of bytes to return (an incomplete line may be returned then).
|
||||
Return an empty string at EOF.
|
||||
"""
|
||||
if self.closed:
|
||||
raise ValueError('I/O operation on closed file')
|
||||
data = ''
|
||||
while '\n' not in data and (size < 0 or len(data) < size):
|
||||
if size < 0:
|
||||
chunk = self.read(1024)
|
||||
else:
|
||||
chunk = self.read(size - len(data))
|
||||
if not chunk:
|
||||
break
|
||||
data += chunk
|
||||
if '\n' in data:
|
||||
data, sep, rest = data.partition('\n')
|
||||
data += sep
|
||||
if self.buf:
|
||||
self.buf = rest + self.buf
|
||||
else:
|
||||
self.buf = rest
|
||||
return data
|
||||
|
||||
def readlines(self, sizehint=-1):
|
||||
"""
|
||||
readlines([size]) -> list of strings, each a line from the file.
|
||||
|
||||
Call readline() repeatedly and return a list of the lines so read.
|
||||
The optional size argument, if given, is an approximate bound on the
|
||||
total number of bytes in the lines returned.
|
||||
"""
|
||||
if self.closed:
|
||||
raise ValueError('I/O operation on closed file')
|
||||
lines = []
|
||||
while True:
|
||||
line = self.readline(sizehint)
|
||||
if not line:
|
||||
break
|
||||
lines.append(line)
|
||||
if sizehint >= 0:
|
||||
sizehint -= len(line)
|
||||
if sizehint <= 0:
|
||||
break
|
||||
return lines
|
||||
|
||||
def close(self):
|
||||
"""
|
||||
close() -> None or (perhaps) an integer. Close the file.
|
||||
|
||||
Sets data attribute .closed to True. A closed file cannot be used for
|
||||
further I/O operations. close() may be called more than once without
|
||||
error. Some kinds of file objects (for example, opened by popen())
|
||||
may return an exit status upon closing.
|
||||
"""
|
||||
self.iterator = None
|
||||
self.closed = True
|
||||
|
||||
|
||||
class FallocateWrapper(object):
|
||||
|
||||
def __init__(self, noop=False):
|
||||
|
||||
@ -27,47 +27,11 @@ from swift.common.direct_client import direct_get_object
|
||||
from swift.common.ring import Ring
|
||||
from swift.common.db import ContainerBroker
|
||||
from swift.common.utils import audit_location_generator, get_logger, \
|
||||
hash_path, config_true_value, validate_sync_to, whataremyips
|
||||
hash_path, config_true_value, validate_sync_to, whataremyips, FileLikeIter
|
||||
from swift.common.daemon import Daemon
|
||||
from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND
|
||||
|
||||
|
||||
class _Iter2FileLikeObject(object):
|
||||
"""
|
||||
Returns an iterator's contents via :func:`read`, making it look like a file
|
||||
object.
|
||||
"""
|
||||
|
||||
def __init__(self, iterator):
|
||||
self.iterator = iterator
|
||||
self._chunk = ''
|
||||
|
||||
def read(self, size=-1):
|
||||
"""
|
||||
read([size]) -> read at most size bytes, returned as a string.
|
||||
|
||||
If the size argument is negative or omitted, read until EOF is reached.
|
||||
Notice that when in non-blocking mode, less data than what was
|
||||
requested may be returned, even if no size parameter was given.
|
||||
"""
|
||||
if size < 0:
|
||||
chunk = self._chunk
|
||||
self._chunk = ''
|
||||
return chunk + ''.join(self.iterator)
|
||||
chunk = self._chunk
|
||||
self._chunk = ''
|
||||
if chunk and len(chunk) <= size:
|
||||
return chunk
|
||||
try:
|
||||
chunk += self.iterator.next()
|
||||
except StopIteration:
|
||||
pass
|
||||
if len(chunk) <= size:
|
||||
return chunk
|
||||
self._chunk = chunk[size:]
|
||||
return chunk[:size]
|
||||
|
||||
|
||||
class ContainerSync(Daemon):
|
||||
"""
|
||||
Daemon to sync syncable containers.
|
||||
@ -409,7 +373,7 @@ class ContainerSync(Daemon):
|
||||
headers['x-timestamp'] = row['created_at']
|
||||
headers['x-container-sync-key'] = sync_key
|
||||
put_object(sync_to, name=row['name'], headers=headers,
|
||||
contents=_Iter2FileLikeObject(body),
|
||||
contents=FileLikeIter(body),
|
||||
proxy=self.proxy)
|
||||
self.container_puts += 1
|
||||
self.logger.increment('puts')
|
||||
|
||||
@ -1325,6 +1325,124 @@ log_name = %(yarr)s'''
|
||||
MagicMock(side_effect=BaseException('test3')))
|
||||
|
||||
|
||||
class TestFileLikeIter(unittest.TestCase):
|
||||
|
||||
def test_iter_file_iter(self):
|
||||
in_iter = ['abc', 'de', 'fghijk', 'l']
|
||||
chunks = []
|
||||
for chunk in utils.FileLikeIter(in_iter):
|
||||
chunks.append(chunk)
|
||||
self.assertEquals(chunks, in_iter)
|
||||
|
||||
def test_next(self):
|
||||
in_iter = ['abc', 'de', 'fghijk', 'l']
|
||||
chunks = []
|
||||
iter_file = utils.FileLikeIter(in_iter)
|
||||
while True:
|
||||
try:
|
||||
chunk = iter_file.next()
|
||||
except StopIteration:
|
||||
break
|
||||
chunks.append(chunk)
|
||||
self.assertEquals(chunks, in_iter)
|
||||
|
||||
def test_read(self):
|
||||
in_iter = ['abc', 'de', 'fghijk', 'l']
|
||||
chunks = []
|
||||
iter_file = utils.FileLikeIter(in_iter)
|
||||
self.assertEquals(iter_file.read(), ''.join(in_iter))
|
||||
|
||||
def test_read_with_size(self):
|
||||
in_iter = ['abc', 'de', 'fghijk', 'l']
|
||||
chunks = []
|
||||
iter_file = utils.FileLikeIter(in_iter)
|
||||
while True:
|
||||
chunk = iter_file.read(2)
|
||||
if not chunk:
|
||||
break
|
||||
self.assertTrue(len(chunk) <= 2)
|
||||
chunks.append(chunk)
|
||||
self.assertEquals(''.join(chunks), ''.join(in_iter))
|
||||
|
||||
def test_read_with_size_zero(self):
|
||||
# makes little sense, but file supports it, so...
|
||||
self.assertEquals(utils.FileLikeIter('abc').read(0), '')
|
||||
|
||||
def test_readline(self):
|
||||
in_iter = ['abc\n', 'd', '\nef', 'g\nh', '\nij\n\nk\n', 'trailing.']
|
||||
lines = []
|
||||
iter_file = utils.FileLikeIter(in_iter)
|
||||
while True:
|
||||
line = iter_file.readline()
|
||||
if not line:
|
||||
break
|
||||
lines.append(line)
|
||||
self.assertEquals(
|
||||
lines,
|
||||
[v if v == 'trailing.' else v + '\n'
|
||||
for v in ''.join(in_iter).split('\n')])
|
||||
|
||||
def test_readline2(self):
|
||||
self.assertEquals(
|
||||
utils.FileLikeIter(['abc', 'def\n']).readline(4),
|
||||
'abcd')
|
||||
|
||||
def test_readline3(self):
|
||||
self.assertEquals(
|
||||
utils.FileLikeIter(['a' * 1111, 'bc\ndef']).readline(),
|
||||
('a' * 1111) + 'bc\n')
|
||||
|
||||
def test_readline_with_size(self):
|
||||
|
||||
in_iter = ['abc\n', 'd', '\nef', 'g\nh', '\nij\n\nk\n', 'trailing.']
|
||||
lines = []
|
||||
iter_file = utils.FileLikeIter(in_iter)
|
||||
while True:
|
||||
line = iter_file.readline(2)
|
||||
if not line:
|
||||
break
|
||||
lines.append(line)
|
||||
self.assertEquals(
|
||||
lines,
|
||||
['ab', 'c\n', 'd\n', 'ef', 'g\n', 'h\n', 'ij', '\n', '\n', 'k\n',
|
||||
'tr', 'ai', 'li', 'ng', '.'])
|
||||
|
||||
def test_readlines(self):
|
||||
in_iter = ['abc\n', 'd', '\nef', 'g\nh', '\nij\n\nk\n', 'trailing.']
|
||||
lines = utils.FileLikeIter(in_iter).readlines()
|
||||
self.assertEquals(
|
||||
lines,
|
||||
[v if v == 'trailing.' else v + '\n'
|
||||
for v in ''.join(in_iter).split('\n')])
|
||||
|
||||
def test_readlines_with_size(self):
|
||||
in_iter = ['abc\n', 'd', '\nef', 'g\nh', '\nij\n\nk\n', 'trailing.']
|
||||
iter_file = utils.FileLikeIter(in_iter)
|
||||
lists_of_lines = []
|
||||
while True:
|
||||
lines = iter_file.readlines(2)
|
||||
if not lines:
|
||||
break
|
||||
lists_of_lines.append(lines)
|
||||
self.assertEquals(
|
||||
lists_of_lines,
|
||||
[['ab'], ['c\n'], ['d\n'], ['ef'], ['g\n'], ['h\n'], ['ij'],
|
||||
['\n', '\n'], ['k\n'], ['tr'], ['ai'], ['li'], ['ng'], ['.']])
|
||||
|
||||
def test_close(self):
|
||||
iter_file = utils.FileLikeIter('abcdef')
|
||||
self.assertEquals(iter_file.next(), 'a')
|
||||
iter_file.close()
|
||||
self.assertTrue(iter_file.closed, True)
|
||||
self.assertRaises(ValueError, iter_file.next)
|
||||
self.assertRaises(ValueError, iter_file.read)
|
||||
self.assertRaises(ValueError, iter_file.readline)
|
||||
self.assertRaises(ValueError, iter_file.readlines)
|
||||
# Just make sure repeated close calls don't raise an Exception
|
||||
iter_file.close()
|
||||
self.assertTrue(iter_file.closed, True)
|
||||
|
||||
|
||||
class TestStatsdLogging(unittest.TestCase):
|
||||
def test_get_logger_statsd_client_not_specified(self):
|
||||
logger = utils.get_logger({}, 'some-name', log_route='some-route')
|
||||
|
||||
@ -66,8 +66,10 @@ class FakeContainerBroker(object):
|
||||
|
||||
class TestContainerSync(unittest.TestCase):
|
||||
|
||||
def test_Iter2FileLikeObject(self):
|
||||
flo = sync._Iter2FileLikeObject(iter(['123', '4567', '89', '0']))
|
||||
def test_FileLikeIter(self):
|
||||
# Retained test to show new FileLikeIter acts just like the removed
|
||||
# _Iter2FileLikeObject did.
|
||||
flo = sync.FileLikeIter(iter(['123', '4567', '89', '0']))
|
||||
expect = '1234567890'
|
||||
|
||||
got = flo.read(2)
|
||||
@ -84,7 +86,7 @@ class TestContainerSync(unittest.TestCase):
|
||||
self.assertEquals(flo.read(), '')
|
||||
self.assertEquals(flo.read(2), '')
|
||||
|
||||
flo = sync._Iter2FileLikeObject(iter(['123', '4567', '89', '0']))
|
||||
flo = sync.FileLikeIter(iter(['123', '4567', '89', '0']))
|
||||
self.assertEquals(flo.read(), '1234567890')
|
||||
self.assertEquals(flo.read(), '')
|
||||
self.assertEquals(flo.read(2), '')
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user