diff --git a/swift/common/bufferedhttp.py b/swift/common/bufferedhttp.py index 61d436976d..402345b3d2 100644 --- a/swift/common/bufferedhttp.py +++ b/swift/common/bufferedhttp.py @@ -51,15 +51,19 @@ class BufferedHTTPResponse(HTTPResponse): method=None): # pragma: no cover self.sock = sock # sock is an eventlet.greenio.GreenSocket - # sock.fd is a socket._socketobject - # sock.fd._sock is a socket._socket object, which is what we want. - self._real_socket = sock.fd._sock + if six.PY2: + # sock.fd is a socket._socketobject + # sock.fd._sock is a _socket.socket object, which is what we want. + self._real_socket = sock.fd._sock + else: + # sock.fd is a socket.socket, which should have a _real_close + self._real_socket = sock.fd self.fp = sock.makefile('rb') self.debuglevel = debuglevel self.strict = strict self._method = method - self.msg = None + self.headers = self.msg = None # from the Status-Line of the response self.version = _UNKNOWN # HTTP-Version @@ -70,7 +74,7 @@ class BufferedHTTPResponse(HTTPResponse): self.chunk_left = _UNKNOWN # bytes left to read in current chunk self.length = _UNKNOWN # number of bytes left in response self.will_close = _UNKNOWN # conn will close at end of response - self._readline_buffer = '' + self._readline_buffer = b'' def expect_response(self): if self.fp: @@ -85,8 +89,15 @@ class BufferedHTTPResponse(HTTPResponse): self.status = status self.reason = reason.strip() self.version = 11 - self.msg = HTTPMessage(self.fp, 0) - self.msg.fp = None + if six.PY2: + # Under py2, HTTPMessage.__init__ reads the headers + # which advances fp + self.msg = HTTPMessage(self.fp, 0) + # immediately kill msg.fp to make sure it isn't read again + self.msg.fp = None + else: + # py3 has a separate helper for it + self.headers = self.msg = httplib.parse_headers(self.fp) def read(self, amt=None): if not self._readline_buffer: @@ -96,7 +107,7 @@ class BufferedHTTPResponse(HTTPResponse): # Unbounded read: send anything we have buffered plus whatever # is left. buffered = self._readline_buffer - self._readline_buffer = '' + self._readline_buffer = b'' return buffered + HTTPResponse.read(self, amt) elif amt <= len(self._readline_buffer): # Bounded read that we can satisfy entirely from our buffer @@ -107,7 +118,7 @@ class BufferedHTTPResponse(HTTPResponse): # Bounded read that wants more bytes than we have smaller_amt = amt - len(self._readline_buffer) buf = self._readline_buffer - self._readline_buffer = '' + self._readline_buffer = b'' return buf + HTTPResponse.read(self, smaller_amt) def readline(self, size=1024): @@ -118,7 +129,7 @@ class BufferedHTTPResponse(HTTPResponse): # # too. # # Yes, it certainly would. - while ('\n' not in self._readline_buffer + while (b'\n' not in self._readline_buffer and len(self._readline_buffer) < size): read_size = size - len(self._readline_buffer) chunk = HTTPResponse.read(self, read_size) @@ -126,7 +137,7 @@ class BufferedHTTPResponse(HTTPResponse): break self._readline_buffer += chunk - line, newline, rest = self._readline_buffer.partition('\n') + line, newline, rest = self._readline_buffer.partition(b'\n') self._readline_buffer = rest return line + newline @@ -139,9 +150,14 @@ class BufferedHTTPResponse(HTTPResponse): you care about has a reference to this socket. """ if self._real_socket: - # this is idempotent; see sock_close in Modules/socketmodule.c in - # the Python source for details. - self._real_socket.close() + if six.PY2: + # this is idempotent; see sock_close in Modules/socketmodule.c + # in the Python source for details. + self._real_socket.close() + else: + # Hopefully this is equivalent? + # TODO: verify that this does everything ^^^^ does for py2 + self._real_socket._real_close() self._real_socket = None self.close() @@ -168,8 +184,10 @@ class BufferedHTTPConnection(HTTPConnection): skip_accept_encoding) def getexpect(self): - response = BufferedHTTPResponse(self.sock, strict=self.strict, - method=self._method) + kwargs = {'method': self._method} + if hasattr(self, 'strict'): + kwargs['strict'] = self.strict + response = BufferedHTTPResponse(self.sock, **kwargs) response.expect_response() return response @@ -205,7 +223,11 @@ def http_connect(ipaddr, port, device, partition, method, path, path = path.encode("utf-8") if isinstance(device, six.text_type): device = device.encode("utf-8") - path = quote('/' + device + '/' + str(partition) + path) + if isinstance(partition, six.text_type): + partition = partition.encode('utf-8') + elif isinstance(partition, six.integer_types): + partition = str(partition).encode('ascii') + path = quote(b'/' + device + b'/' + partition + path) return http_connect_raw( ipaddr, port, method, path, headers, query_string, ssl) diff --git a/test/unit/common/test_bufferedhttp.py b/test/unit/common/test_bufferedhttp.py index 26dce69f30..7e0c9ec03e 100644 --- a/test/unit/common/test_bufferedhttp.py +++ b/test/unit/common/test_bufferedhttp.py @@ -14,9 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import mock - import unittest - import socket from eventlet import spawn, Timeout @@ -53,28 +51,31 @@ class TestBufferedHTTP(unittest.TestCase): try: with Timeout(3): sock, addr = bindsock.accept() - fp = sock.makefile() - fp.write('HTTP/1.1 200 OK\r\nContent-Length: 8\r\n\r\n' - 'RESPONSE') + fp = sock.makefile('rwb') + fp.write(b'HTTP/1.1 200 OK\r\nContent-Length: 8\r\n\r\n' + b'RESPONSE') fp.flush() + line = fp.readline() + path = b'/dev/' + expected_par + b'/path/..%25/?omg&no=%7f' self.assertEqual( - fp.readline(), - 'PUT /dev/%s/path/..%%25/?omg&no=%%7f HTTP/1.1\r\n' % - expected_par) + line, + b'PUT ' + path + b' HTTP/1.1\r\n') headers = {} line = fp.readline() - while line and line != '\r\n': - headers[line.split(':')[0].lower()] = \ - line.split(':')[1].strip() + while line and line != b'\r\n': + headers[line.split(b':')[0].lower()] = \ + line.split(b':')[1].strip() line = fp.readline() - self.assertEqual(headers['content-length'], '7') - self.assertEqual(headers['x-header'], 'value') - self.assertEqual(fp.readline(), 'REQUEST\r\n') + self.assertEqual(headers[b'content-length'], b'7') + self.assertEqual(headers[b'x-header'], b'value') + self.assertEqual(fp.readline(), b'REQUEST\r\n') except BaseException as err: return err return None - for par in ('par', 1357): - event = spawn(accept, par) + for spawn_par, par in ( + (b'par', b'par'), (b'up%C3%A5r', u'up\xe5r'), + (b'%C3%BCpar', b'\xc3\xbcpar'), (b'1357', 1357)): + event = spawn(accept, spawn_par) try: with Timeout(3): conn = bufferedhttp.http_connect( @@ -83,7 +84,7 @@ class TestBufferedHTTP(unittest.TestCase): 'content-length': 7, 'x-header': 'value'}, query_string='omg&no=%7f') - conn.send('REQUEST\r\n') + conn.send(b'REQUEST\r\n') self.assertTrue(conn.sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY)) resp = conn.getresponse() @@ -91,7 +92,7 @@ class TestBufferedHTTP(unittest.TestCase): conn.close() self.assertEqual(resp.status, 200) self.assertEqual(resp.reason, 'OK') - self.assertEqual(body, 'RESPONSE') + self.assertEqual(body, b'RESPONSE') finally: err = event.wait() if err: diff --git a/tox.ini b/tox.ini index ba43477ef5..e38e5c8c61 100644 --- a/tox.ini +++ b/tox.ini @@ -43,6 +43,7 @@ commands = test/unit/common/middleware/test_healthcheck.py \ test/unit/common/middleware/test_proxy_logging.py \ test/unit/common/ring \ + test/unit/common/test_bufferedhttp.py \ test/unit/common/test_constraints.py \ test/unit/common/test_daemon.py \ test/unit/common/test_exceptions.py \