openstack/swift
Code Issues Proposed changes

Stop monkey-patching mimetools

Browse Source 
You could *try* doing something similar to what we were doing
there over in email.message for py3, but you would end up
breaking pkg_resources (and therefor entrypoints) in the
process.

Drive-by: have mem_diskfile implement more of the diskfile API.

Change-Id: I1ece4b4500ce37408799ee634ed6d7832fb7b721
This commit is contained in:
Tim Burke 2019-03-01 14:00:35 -08:00
parent 52211e3d73
commit e5eb673ccb
7 changed files with 67 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
swift/common/wsgi.py
View File

@ -32,8 +32,6 @@ from eventlet.green import socket, ssl, os as green_os
import six
from six import BytesIO
from six import StringIO
if six.PY2:
import mimetools
from swift.common import utils, constraints
from swift.common.storage_policy import BindPortsCache
@ -147,31 +145,6 @@ def wrap_conf_type(f):
appconfig = wrap_conf_type(loadwsgi.appconfig)
def monkey_patch_mimetools():
"""
mimetools.Message defaults content-type to "text/plain"
This changes it to default to None, so we can detect missing headers.
"""
if six.PY3:
# The mimetools has been removed from Python 3
return
orig_parsetype = mimetools.Message.parsetype
def parsetype(self):
if not self.typeheader:
self.type = None
self.maintype = None
self.subtype = None
self.plisttext = ''
else:
orig_parsetype(self)
parsetype.patched = True
if not getattr(mimetools.Message.parsetype, 'patched', None):
mimetools.Message.parsetype = parsetype
def get_socket(conf):
"""Bind socket to bind ip:port in conf
@ -447,6 +420,18 @@ class SwiftHttpProtocol(wsgi.HttpProtocol):
# versions the output from error is same as info anyway
self.server.log.info('ERROR WSGI: ' + f, *a)
class MessageClass(wsgi.HttpProtocol.MessageClass):
'''Subclass to see when the client didn't provide a Content-Type'''
# for py2:
def parsetype(self):
if self.typeheader is None:
self.typeheader = ''
wsgi.HttpProtocol.MessageClass.parsetype(self)
# for py3:
def get_default_type(self):
return ''
class SwiftHttpProxiedProtocol(SwiftHttpProtocol):
"""
@ -1155,7 +1140,6 @@ def _initrp(conf_path, app_section, *args, **kwargs):
if config_true_value(conf.get('disable_fallocate', 'no')):
disable_fallocate()
monkey_patch_mimetools()
return (conf, logger, log_name)

3
swift/obj/mem_diskfile.py
View File

@ -412,6 +412,9 @@ class DiskFile(object):
raise DiskFileNotOpen()
return self._metadata
get_datafile_metadata = get_metadata
get_metafile_metadata = get_metadata
def read_metadata(self, current_time=None):
"""
Return the metadata for an object.

5
test/functional/__init__.py
View File

@ -53,8 +53,7 @@ from test.unit import SkipTest
from swift.common import constraints, utils, ring, storage_policy
from swift.common.ring import Ring
from swift.common.wsgi import (
monkey_patch_mimetools, loadapp, SwiftHttpProtocol)
from swift.common.wsgi import loadapp, SwiftHttpProtocol
from swift.common.utils import config_true_value, split_path
from swift.account import server as account_server
from swift.container import server as container_server
@ -493,8 +492,6 @@ def in_process_setup(the_object_server=object_server):
swift_conf_src = _in_process_find_conf_file(conf_src_dir, 'swift.conf')
_info('Using swift config from %s' % swift_conf_src)
monkey_patch_mimetools()
global _testdir
_testdir = os.path.join(mkdtemp(), 'tmp_functional')
utils.mkdirs(_testdir)

51
test/unit/common/test_wsgi.py
View File

@ -27,12 +27,8 @@ import types
import eventlet.wsgi
import six
from six import BytesIO
from six import StringIO
from six.moves.urllib.parse import quote
if six.PY2:
import mimetools
import mock
@ -69,53 +65,6 @@ def _fake_rings(tmpdir):
class TestWSGI(unittest.TestCase):
"""Tests for swift.common.wsgi"""
def setUp(self):
if six.PY2:
self._orig_parsetype = mimetools.Message.parsetype
def tearDown(self):
if six.PY2:
mimetools.Message.parsetype = self._orig_parsetype
@unittest.skipIf(six.PY3, "test specific to Python 2")
def test_monkey_patch_mimetools(self):
sio = StringIO('blah')
self.assertEqual(mimetools.Message(sio).type, 'text/plain')
sio = StringIO('blah')
self.assertEqual(mimetools.Message(sio).plisttext, '')
sio = StringIO('blah')
self.assertEqual(mimetools.Message(sio).maintype, 'text')
sio = StringIO('blah')
self.assertEqual(mimetools.Message(sio).subtype, 'plain')
sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
self.assertEqual(mimetools.Message(sio).type, 'text/html')
sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
self.assertEqual(mimetools.Message(sio).plisttext,
'; charset=ISO-8859-4')
sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
self.assertEqual(mimetools.Message(sio).maintype, 'text')
sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
self.assertEqual(mimetools.Message(sio).subtype, 'html')
wsgi.monkey_patch_mimetools()
sio = StringIO('blah')
self.assertIsNone(mimetools.Message(sio).type)
sio = StringIO('blah')
self.assertEqual(mimetools.Message(sio).plisttext, '')
sio = StringIO('blah')
self.assertIsNone(mimetools.Message(sio).maintype)
sio = StringIO('blah')
self.assertIsNone(mimetools.Message(sio).subtype)
sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
self.assertEqual(mimetools.Message(sio).type, 'text/html')
sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
self.assertEqual(mimetools.Message(sio).plisttext,
'; charset=ISO-8859-4')
sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
self.assertEqual(mimetools.Message(sio).maintype, 'text')
sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
self.assertEqual(mimetools.Message(sio).subtype, 'html')
def test_init_request_processor(self):
config = """
[DEFAULT]

34
test/unit/helpers.py
View File

@ -40,6 +40,7 @@ from swift.common.storage_policy import StoragePolicy, ECStoragePolicy
from swift.common.middleware import listing_formats, proxy_logging
from swift.common import utils
from swift.common.utils import mkdirs, normalize_timestamp, NullLogger
from swift.common.wsgi import SwiftHttpProtocol
from swift.container import server as container_server
from swift.obj import server as object_server
from swift.proxy import server as proxy_server
@ -212,17 +213,28 @@ def setup_servers(the_object_server=object_server, extra_conf=None):
nl = NullLogger()
logging_prosv = proxy_logging.ProxyLoggingMiddleware(
listing_formats.ListingFilter(prosrv), conf, logger=prosrv.logger)
prospa = spawn(wsgi.server, prolis, logging_prosv, nl)
acc1spa = spawn(wsgi.server, acc1lis, acc1srv, nl)
acc2spa = spawn(wsgi.server, acc2lis, acc2srv, nl)
con1spa = spawn(wsgi.server, con1lis, con1srv, nl)
con2spa = spawn(wsgi.server, con2lis, con2srv, nl)
obj1spa = spawn(wsgi.server, obj1lis, obj1srv, nl)
obj2spa = spawn(wsgi.server, obj2lis, obj2srv, nl)
obj3spa = spawn(wsgi.server, obj3lis, obj3srv, nl)
obj4spa = spawn(wsgi.server, obj4lis, obj4srv, nl)
obj5spa = spawn(wsgi.server, obj5lis, obj5srv, nl)
obj6spa = spawn(wsgi.server, obj6lis, obj6srv, nl)
prospa = spawn(wsgi.server, prolis, logging_prosv, nl,
protocol=SwiftHttpProtocol)
acc1spa = spawn(wsgi.server, acc1lis, acc1srv, nl,
protocol=SwiftHttpProtocol)
acc2spa = spawn(wsgi.server, acc2lis, acc2srv, nl,
protocol=SwiftHttpProtocol)
con1spa = spawn(wsgi.server, con1lis, con1srv, nl,
protocol=SwiftHttpProtocol)
con2spa = spawn(wsgi.server, con2lis, con2srv, nl,
protocol=SwiftHttpProtocol)
obj1spa = spawn(wsgi.server, obj1lis, obj1srv, nl,
protocol=SwiftHttpProtocol)
obj2spa = spawn(wsgi.server, obj2lis, obj2srv, nl,
protocol=SwiftHttpProtocol)
obj3spa = spawn(wsgi.server, obj3lis, obj3srv, nl,
protocol=SwiftHttpProtocol)
obj4spa = spawn(wsgi.server, obj4lis, obj4srv, nl,
protocol=SwiftHttpProtocol)
obj5spa = spawn(wsgi.server, obj5lis, obj5srv, nl,
protocol=SwiftHttpProtocol)
obj6spa = spawn(wsgi.server, obj6lis, obj6srv, nl,
protocol=SwiftHttpProtocol)
context["test_coros"] = \
(prospa, acc1spa, acc2spa, con1spa, con2spa, obj1spa, obj2spa, obj3spa,
obj4spa, obj5spa, obj6spa)

57
test/unit/proxy/test_server.py
View File

@ -71,7 +71,7 @@ from swift.common import utils, constraints
from swift.common.utils import hash_path, storage_directory, \
parse_content_type, parse_mime_headers, \
iter_multipart_mime_documents, public, mkdirs, NullLogger
from swift.common.wsgi import monkey_patch_mimetools, loadapp, ConfigString
from swift.common.wsgi import loadapp, ConfigString
from swift.proxy.controllers import base as proxy_base
from swift.proxy.controllers.base import get_cache_key, cors_validation, \
get_account_info, get_container_info
@ -97,7 +97,6 @@ def do_setup(object_server):
# setup test context and break out some globals for convenience
global _test_context, _testdir, _test_servers, _test_sockets, \
_test_POLICIES
monkey_patch_mimetools()
_test_context = setup_servers(object_server)
_testdir = _test_context["testdir"]
_test_servers = _test_context["test_servers"]
@ -3269,37 +3268,35 @@ class TestReplicatedObjectController(
self.assertNotEqual(last_modified_put, last_modified_head)
_do_conditional_GET_checks(last_modified_head)
@unpatch_policies
def test_PUT_auto_content_type(self):
with save_globals():
controller = ReplicatedObjectController(
self.app, 'account', 'container', 'object')
prolis = _test_sockets[0]
def test_content_type(filename, expected):
# The three responses here are for account_info() (HEAD to
# account server), container_info() (HEAD to container server)
# and three calls to _connect_put_node() (PUT to three object
# servers)
set_http_connect(201, 201, 201, 201, 201,
give_content_type=lambda content_type:
self.assertEqual(content_type,
next(expected)))
# We need into include a transfer-encoding to get past
# constraints.check_object_creation()
req = Request.blank('/v1/a/c/%s' % filename, {},
headers={'transfer-encoding': 'chunked'})
self.app.update_request(req)
self.app.memcache.store = {}
res = controller.PUT(req)
# If we don't check the response here we could miss problems
# in PUT()
self.assertEqual(res.status_int, 201)
def do_test(ext, content_type):
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile('rwb')
fd.write(b'PUT /v1/a/c/o.%s HTTP/1.1\r\n'
b'Host: localhost\r\n'
b'X-Storage-Token: t\r\nContent-Length: 0\r\n\r\n' %
ext.encode())
fd.flush()
headers = readuntil2crlfs(fd)
exp = b'HTTP/1.1 201'
self.assertEqual(headers[:len(exp)], exp)
test_content_type('test.jpg', iter(['', '', 'image/jpeg',
'image/jpeg', 'image/jpeg']))
test_content_type('test.html', iter(['', '', 'text/html',
'text/html', 'text/html']))
test_content_type('test.css', iter(['', '', 'text/css',
'text/css', 'text/css']))
fd.write(b'GET /v1/a/c/o.%s HTTP/1.1\r\n'
b'Host: localhost\r\nConnection: close\r\n'
b'X-Storage-Token: t\r\n\r\n' % ext.encode())
fd.flush()
headers = readuntil2crlfs(fd)
exp = b'HTTP/1.1 200'
self.assertIn(b'Content-Type: %s' % content_type.encode(),
headers.split(b'\r\n'))
sock.close()
do_test('jpg', 'image/jpeg')
do_test('html', 'text/html')
do_test('css', 'text/css')
def test_custom_mime_types_files(self):
swift_dir = mkdtemp()

3
test/unit/proxy/test_sysmeta.py
View File

@ -24,7 +24,7 @@ from swift.common.middleware.copy import ServerSideCopyMiddleware
from swift.common.storage_policy import StoragePolicy
from swift.common.swob import Request
from swift.common.utils import mkdirs, split_path
from swift.common.wsgi import monkey_patch_mimetools, WSGIContext
from swift.common.wsgi import WSGIContext
from swift.obj import server as object_server
from swift.proxy import server as proxy
import swift.proxy.controllers
@ -138,7 +138,6 @@ class TestObjectSysmeta(unittest.TestCase):
account_ring=FakeRing(replicas=1),
container_ring=FakeRing(replicas=1))
self.copy_app = ServerSideCopyMiddleware(self.app, {})
monkey_patch_mimetools()
self.tmpdir = mkdtemp()
self.testdir = os.path.join(self.tmpdir,
'tmp_test_object_server_ObjectController')