openstack/swift
Code Issues Proposed changes

Consistently use io.BytesIO

Browse Source 
Change-Id: Ic41b37ac75b5596a8307c4962be86f2a4b0d9731
This commit is contained in:
Tim Burke 2019-10-14 14:03:16 -07:00 committed by Thiago da Silva
parent 1c0661e920
commit d270596b67
22 changed files with 73 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
swift/common/swob.py
View File

@ -44,9 +44,9 @@ import re
import random
import functools
import inspect
from io import BytesIO
import six
from six import BytesIO
from six import StringIO
from six.moves import urllib

3
swift/common/wsgi.py
View File

@ -29,8 +29,9 @@ import eventlet.debug
from eventlet import greenio, GreenPool, sleep, wsgi, listen, Timeout
from paste.deploy import loadwsgi
from eventlet.green import socket, ssl, os as green_os
from io import BytesIO
import six
from six import BytesIO
from six import StringIO
from six.moves import configparser

6
test/functional/swift_test_client.py
View File

@ -843,7 +843,7 @@ class File(Base):
block_size = 4096
if isinstance(data, bytes):
data = six.BytesIO(data)
data = io.BytesIO(data)
checksum = hashlib.md5()
buff = data.read(block_size)
@ -1186,7 +1186,7 @@ class File(Base):
if not self.write(data, hdrs=hdrs, parms=parms, cfg=cfg):
raise ResponseError(self.conn.response, 'PUT',
self.conn.make_path(self.path))
self.md5 = self.compute_md5sum(six.BytesIO(data))
self.md5 = self.compute_md5sum(io.BytesIO(data))
return data
def write_random_return_resp(self, size=None, hdrs=None, parms=None,
@ -1203,7 +1203,7 @@ class File(Base):
return_resp=True)
if not resp:
raise ResponseError(self.conn.response)
self.md5 = self.compute_md5sum(six.BytesIO(data))
self.md5 = self.compute_md5sum(io.BytesIO(data))
return resp
def post(self, hdrs=None, parms=None, cfg=None, return_resp=False):

3
test/functional/tests.py
View File

@ -16,6 +16,7 @@
from datetime import datetime
import hashlib
import io
import locale
import random
import six
@ -2609,7 +2610,7 @@ class TestFile(Base):
def testEtagResponse(self):
file_item = self.env.container.file(Utils.create_name())
data = six.BytesIO(file_item.write_random(512))
data = io.BytesIO(file_item.write_random(512))
etag = File.compute_md5sum(data)
headers = dict((h.lower(), v)

2
test/unit/__init__.py
View File

@ -38,10 +38,10 @@ import json
import random
import errno
import xattr
from io import BytesIO
import six
import six.moves.cPickle as pickle
from six import BytesIO
from six.moves import range
from six.moves.http_client import HTTPException

2
test/unit/account/test_server.py
View File

@ -23,9 +23,9 @@ from shutil import rmtree
from test.unit import FakeLogger
import itertools
import random
from io import BytesIO
import json
from six import BytesIO
from six import StringIO
import xml.dom.minidom

2
test/unit/common/middleware/s3api/test_acl.py
View File

@ -17,7 +17,7 @@ import base64
import unittest
import mock
from six import BytesIO
from io import BytesIO
from hashlib import md5
from swift.common.swob import Request, HTTPAccepted

2
test/unit/common/middleware/s3api/test_s3request.py
View File

@ -17,7 +17,7 @@ import hashlib
from mock import patch, MagicMock
import unittest
from six import BytesIO
from io import BytesIO
from swift.common import swob
from swift.common.swob import Request, HTTPNoContent

2
test/unit/common/middleware/test_bulk.py
View File

@ -22,7 +22,7 @@ import tarfile
import zlib
import mock
import six
from six import BytesIO
from io import BytesIO
from shutil import rmtree
from tempfile import mkdtemp
from eventlet import sleep

2
test/unit/common/middleware/test_formpost.py
View File

@ -19,7 +19,7 @@ from hashlib import sha1
from time import time
import six
from six import BytesIO
from io import BytesIO
from swift.common.swob import Request, Response
from swift.common.middleware import tempauth, formpost

6
test/unit/common/middleware/test_proxy_logging.py
View File

@ -13,20 +13,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
import unittest
from io import BytesIO
from logging.handlers import SysLogHandler
import mock
import six
from six import BytesIO
from six.moves.urllib.parse import unquote
from test.unit import FakeLogger
from swift.common.utils import get_logger, split_path
from swift.common.middleware import proxy_logging
from swift.common.swob import Request, Response
from swift.common import constraints
from swift.common.storage_policy import StoragePolicy
from test.unit import FakeLogger
from test.unit import patch_policies
from test.unit.common.middleware.helpers import FakeAppThatExcepts

3
test/unit/common/middleware/test_slo.py
View File

@ -23,8 +23,7 @@ import unittest
from mock import patch
import six
from six import BytesIO
from six.moves import range
from io import BytesIO
from swift.common import swob, utils
from swift.common.header_key_dict import HeaderKeyDict

2
test/unit/common/middleware/test_xprofile.py
View File

@ -19,7 +19,7 @@ import shutil
import tempfile
import unittest
from six import BytesIO
from io import BytesIO
from swift import gettext_ as _
from swift.common.swob import Request, Response

16
test/unit/common/test_direct_client.py
View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import io
import json
import unittest
import os
@ -22,7 +23,6 @@ import time
import pickle
import mock
import six
from six.moves import urllib
from swift.common import direct_client
@ -72,7 +72,7 @@ class FakeConn(object):
return self.resp_headers.items()
def read(self, amt=None):
if isinstance(self.body, six.BytesIO):
if isinstance(self.body, io.BytesIO):
return self.body.read(amt)
elif amt is None:
return self.body
@ -628,7 +628,7 @@ class TestDirectClient(unittest.TestCase):
important_timestamp)
def test_direct_get_object(self):
contents = six.BytesIO(b'123456')
contents = io.BytesIO(b'123456')
with mocked_http_conn(200, body=contents) as conn:
resp_header, obj_body = direct_client.direct_get_object(
@ -654,7 +654,7 @@ class TestDirectClient(unittest.TestCase):
self.assertTrue('GET' in str(raised.exception))
def test_direct_get_object_chunks(self):
contents = six.BytesIO(b'123456')
contents = io.BytesIO(b'123456')
with mocked_http_conn(200, body=contents) as conn:
resp_header, obj_body = direct_client.direct_get_object(
@ -774,7 +774,7 @@ class TestDirectClient(unittest.TestCase):
self._test_direct_get_suffix_hashes_fail(507)
def test_direct_put_object_with_content_length(self):
contents = six.BytesIO(b'123456')
contents = io.BytesIO(b'123456')
with mocked_http_conn(200) as conn:
resp = direct_client.direct_put_object(
@ -787,7 +787,7 @@ class TestDirectClient(unittest.TestCase):
self.assertEqual(md5(b'123456').hexdigest(), resp)
def test_direct_put_object_fail(self):
contents = six.BytesIO(b'123456')
contents = io.BytesIO(b'123456')
with mocked_http_conn(500) as conn:
with self.assertRaises(ClientException) as raised:
@ -801,7 +801,7 @@ class TestDirectClient(unittest.TestCase):
self.assertEqual(raised.exception.http_status, 500)
def test_direct_put_object_chunked(self):
contents = six.BytesIO(b'123456')
contents = io.BytesIO(b'123456')
with mocked_http_conn(200) as conn:
resp = direct_client.direct_put_object(
@ -829,7 +829,7 @@ class TestDirectClient(unittest.TestCase):
self.assertEqual(md5(b'0\r\n\r\n').hexdigest(), resp)
def test_direct_put_object_header_content_length(self):
contents = six.BytesIO(b'123456')
contents = io.BytesIO(b'123456')
stub_headers = HeaderKeyDict({
'Content-Length': '6'})

5
test/unit/common/test_internal_client.py
View File

@ -17,11 +17,12 @@ import json
import mock
import unittest
import zlib
from textwrap import dedent
import os
from io import BytesIO
from textwrap import dedent
import six
from six import BytesIO
from six.moves import range, zip_longest
from six.moves.urllib.parse import quote, parse_qsl
from test.unit import FakeLogger

3
test/unit/common/test_swob.py
View File

@ -20,8 +20,9 @@ import unittest
import re
import time
from io import BytesIO
import six
from six import BytesIO
from six.moves.urllib.parse import quote
import swift.common.swob

3
test/unit/common/test_utils.py
View File

@ -46,7 +46,7 @@ import math
import inspect
import six
from six import BytesIO, StringIO
from six import StringIO
from six.moves.queue import Queue, Empty
from six.moves import http_client
from six.moves import range
@ -59,6 +59,7 @@ import fcntl
import shutil
from getpass import getuser
from io import BytesIO
from shutil import rmtree
from functools import partial
from tempfile import TemporaryFile, NamedTemporaryFile, mkdtemp

8
test/unit/common/test_wsgi.py
View File

@ -21,14 +21,14 @@ import logging
import socket
import unittest
import os
from textwrap import dedent
from collections import defaultdict
import types
import eventlet.wsgi
from collections import defaultdict
from io import BytesIO
from textwrap import dedent
import six
from six import BytesIO
from six.moves.urllib.parse import quote
import mock

6
test/unit/container/test_server.py
View File

@ -20,18 +20,18 @@ import posix
import mock
import unittest
import itertools
import time
import random
from contextlib import contextmanager
from io import BytesIO
from shutil import rmtree
from tempfile import mkdtemp
from test.unit import make_timestamp_iter, mock_timestamp_now
from xml.dom import minidom
import time
import random
from eventlet import spawn, Timeout
import json
import six
from six import BytesIO
from six import StringIO
from swift import __version__ as swift_version

43
test/unit/obj/test_ssync_receiver.py
View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import io
import os
import shutil
import tempfile
@ -494,10 +495,10 @@ class TestReceiver(unittest.TestCase):
def test_SSYNC_Exception(self):
class _Wrapper(six.BytesIO):
class _Wrapper(io.BytesIO):
def __init__(self, value):
six.BytesIO.__init__(self, value)
io.BytesIO.__init__(self, value)
self.mock_socket = mock.MagicMock()
def get_socket(self):
@ -529,10 +530,10 @@ class TestReceiver(unittest.TestCase):
def test_SSYNC_Exception_Exception(self):
class _Wrapper(six.BytesIO):
class _Wrapper(io.BytesIO):
def __init__(self, value):
six.BytesIO.__init__(self, value)
io.BytesIO.__init__(self, value)
self.mock_socket = mock.MagicMock()
def get_socket(self):
@ -565,14 +566,14 @@ class TestReceiver(unittest.TestCase):
def test_MISSING_CHECK_timeout(self):
class _Wrapper(six.BytesIO):
class _Wrapper(io.BytesIO):
def __init__(self, value):
six.BytesIO.__init__(self, value)
io.BytesIO.__init__(self, value)
self.mock_socket = mock.MagicMock()
def readline(self, sizehint=-1):
line = six.BytesIO.readline(self)
line = io.BytesIO.readline(self)
if line.startswith(b'hash'):
eventlet.sleep(0.1)
return line
@ -607,14 +608,14 @@ class TestReceiver(unittest.TestCase):
def test_MISSING_CHECK_other_exception(self):
class _Wrapper(six.BytesIO):
class _Wrapper(io.BytesIO):
def __init__(self, value):
six.BytesIO.__init__(self, value)
io.BytesIO.__init__(self, value)
self.mock_socket = mock.MagicMock()
def readline(self, sizehint=-1):
line = six.BytesIO.readline(self)
line = io.BytesIO.readline(self)
if line.startswith(b'hash'):
raise Exception('test exception')
return line
@ -1038,14 +1039,14 @@ class TestReceiver(unittest.TestCase):
def test_UPDATES_timeout(self):
class _Wrapper(six.BytesIO):
class _Wrapper(io.BytesIO):
def __init__(self, value):
six.BytesIO.__init__(self, value)
io.BytesIO.__init__(self, value)
self.mock_socket = mock.MagicMock()
def readline(self, sizehint=-1):
line = six.BytesIO.readline(self)
line = io.BytesIO.readline(self)
if line.startswith(b'DELETE'):
eventlet.sleep(0.1)
return line
@ -1085,14 +1086,14 @@ class TestReceiver(unittest.TestCase):
def test_UPDATES_other_exception(self):
class _Wrapper(six.BytesIO):
class _Wrapper(io.BytesIO):
def __init__(self, value):
six.BytesIO.__init__(self, value)
io.BytesIO.__init__(self, value)
self.mock_socket = mock.MagicMock()
def readline(self, sizehint=-1):
line = six.BytesIO.readline(self)
line = io.BytesIO.readline(self)
if line.startswith(b'DELETE'):
raise Exception('test exception')
return line
@ -1131,10 +1132,10 @@ class TestReceiver(unittest.TestCase):
def test_UPDATES_no_problems_no_hard_disconnect(self):
class _Wrapper(six.BytesIO):
class _Wrapper(io.BytesIO):
def __init__(self, value):
six.BytesIO.__init__(self, value)
io.BytesIO.__init__(self, value)
self.mock_socket = mock.MagicMock()
def get_socket(self):
@ -1981,13 +1982,13 @@ class TestReceiver(unittest.TestCase):
request.read_body = request.environ['wsgi.input'].read(2)
return swob.HTTPInternalServerError()
class _IgnoreReadlineHint(six.BytesIO):
class _IgnoreReadlineHint(io.BytesIO):
def __init__(self, value):
six.BytesIO.__init__(self, value)
io.BytesIO.__init__(self, value)
def readline(self, hint=-1):
return six.BytesIO.readline(self)
return io.BytesIO.readline(self)
self.controller.PUT = _PUT
self.controller.network_chunk_size = 2

15
test/unit/obj/test_ssync_sender.py
View File

@ -12,6 +12,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import io
import os
import time
import unittest
@ -60,7 +61,7 @@ class FakeResponse(ssync_sender.SsyncBufferedHTTPResponse):
if not six.PY2:
chunk_body = chunk_body.encode('ascii')
if chunk_body:
self.fp = six.BytesIO(
self.fp = io.BytesIO(
b'%x\r\n%s\r\n0\r\n\r\n' % (len(chunk_body), chunk_body))
self.ssync_response_buffer = b''
self.ssync_response_chunk_left = 0
@ -691,39 +692,39 @@ class TestSender(BaseTest):
def test_readline_at_start_of_chunk(self):
response = FakeResponse()
response.fp = six.BytesIO(b'2\r\nx\n\r\n')
response.fp = io.BytesIO(b'2\r\nx\n\r\n')
self.assertEqual(response.readline(), b'x\n')
def test_readline_chunk_with_extension(self):
response = FakeResponse()
response.fp = six.BytesIO(
response.fp = io.BytesIO(
b'2 ; chunk=extension\r\nx\n\r\n')
self.assertEqual(response.readline(), b'x\n')
def test_readline_broken_chunk(self):
response = FakeResponse()
response.fp = six.BytesIO(b'q\r\nx\n\r\n')
response.fp = io.BytesIO(b'q\r\nx\n\r\n')
self.assertRaises(
exceptions.ReplicationException, response.readline)
self.assertTrue(response.close_called)
def test_readline_terminated_chunk(self):
response = FakeResponse()
response.fp = six.BytesIO(b'b\r\nnot enough')
response.fp = io.BytesIO(b'b\r\nnot enough')
self.assertRaises(
exceptions.ReplicationException, response.readline)
self.assertTrue(response.close_called)
def test_readline_all(self):
response = FakeResponse()
response.fp = six.BytesIO(b'2\r\nx\n\r\n0\r\n\r\n')
response.fp = io.BytesIO(b'2\r\nx\n\r\n0\r\n\r\n')
self.assertEqual(response.readline(), b'x\n')
self.assertEqual(response.readline(), b'')
self.assertEqual(response.readline(), b'')
def test_readline_all_trailing_not_newline_termed(self):
response = FakeResponse()
response.fp = six.BytesIO(
response.fp = io.BytesIO(
b'2\r\nx\n\r\n3\r\n123\r\n0\r\n\r\n')
self.assertEqual(response.readline(), b'x\n')
self.assertEqual(response.readline(), b'123')

3
test/unit/proxy/test_server.py
View File

@ -46,8 +46,9 @@ import uuid
import mock
from eventlet import sleep, spawn, wsgi, Timeout, debug
from eventlet.green import httplib
from io import BytesIO
import six
from six import BytesIO
from six.moves import range
from six.moves.urllib.parse import quote, parse_qsl