Stop monkey-patching mimetools

You could *try* doing something similar to what we were doing
there over in email.message for py3, but you would end up
breaking pkg_resources (and therefor entrypoints) in the
process.

Drive-by: have mem_diskfile implement more of the diskfile API.

Change-Id: I1ece4b4500ce37408799ee634ed6d7832fb7b721

swift/common/wsgi.py

@@ -32,8 +32,6 @@ from eventlet.green import socket, ssl, os as green_os
 import six
 from six import BytesIO
 from six import StringIO
-if six.PY2:
-    import mimetools
 
 from swift.common import utils, constraints
 from swift.common.storage_policy import BindPortsCache
@@ -147,31 +145,6 @@ def wrap_conf_type(f):
 appconfig = wrap_conf_type(loadwsgi.appconfig)
 
 
-def monkey_patch_mimetools():
-    """
-    mimetools.Message defaults content-type to "text/plain"
-    This changes it to default to None, so we can detect missing headers.
-    """
-    if six.PY3:
-        # The mimetools has been removed from Python 3
-        return
-
-    orig_parsetype = mimetools.Message.parsetype
-
-    def parsetype(self):
-        if not self.typeheader:
-            self.type = None
-            self.maintype = None
-            self.subtype = None
-            self.plisttext = ''
-        else:
-            orig_parsetype(self)
-    parsetype.patched = True
-
-    if not getattr(mimetools.Message.parsetype, 'patched', None):
-        mimetools.Message.parsetype = parsetype
-
-
 def get_socket(conf):
     """Bind socket to bind ip:port in conf
 
@@ -447,6 +420,18 @@ class SwiftHttpProtocol(wsgi.HttpProtocol):
             # versions the output from error is same as info anyway
             self.server.log.info('ERROR WSGI: ' + f, *a)
 
+    class MessageClass(wsgi.HttpProtocol.MessageClass):
+        '''Subclass to see when the client didn't provide a Content-Type'''
+        # for py2:
+        def parsetype(self):
+            if self.typeheader is None:
+                self.typeheader = ''
+            wsgi.HttpProtocol.MessageClass.parsetype(self)
+
+        # for py3:
+        def get_default_type(self):
+            return ''
+
 
 class SwiftHttpProxiedProtocol(SwiftHttpProtocol):
     """
@@ -1155,7 +1140,6 @@ def _initrp(conf_path, app_section, *args, **kwargs):
     if config_true_value(conf.get('disable_fallocate', 'no')):
         disable_fallocate()
 
-    monkey_patch_mimetools()
     return (conf, logger, log_name)
 
 

swift/obj/mem_diskfile.py

@@ -412,6 +412,9 @@ class DiskFile(object):
             raise DiskFileNotOpen()
         return self._metadata
 
+    get_datafile_metadata = get_metadata
+    get_metafile_metadata = get_metadata
+
     def read_metadata(self, current_time=None):
         """
         Return the metadata for an object.

test/functional/__init__.py

@@ -53,8 +53,7 @@ from test.unit import SkipTest
 
 from swift.common import constraints, utils, ring, storage_policy
 from swift.common.ring import Ring
-from swift.common.wsgi import (
+from swift.common.wsgi import loadapp, SwiftHttpProtocol
-    monkey_patch_mimetools, loadapp, SwiftHttpProtocol)
 from swift.common.utils import config_true_value, split_path
 from swift.account import server as account_server
 from swift.container import server as container_server
@@ -493,8 +492,6 @@ def in_process_setup(the_object_server=object_server):
     swift_conf_src = _in_process_find_conf_file(conf_src_dir, 'swift.conf')
     _info('Using swift config from %s' % swift_conf_src)
 
-    monkey_patch_mimetools()
-
     global _testdir
     _testdir = os.path.join(mkdtemp(), 'tmp_functional')
     utils.mkdirs(_testdir)

test/unit/common/test_wsgi.py

@@ -27,12 +27,8 @@ import types
 
 import eventlet.wsgi
 
-import six
 from six import BytesIO
-from six import StringIO
 from six.moves.urllib.parse import quote
-if six.PY2:
-    import mimetools
 
 import mock
 
@@ -69,53 +65,6 @@ def _fake_rings(tmpdir):
 class TestWSGI(unittest.TestCase):
     """Tests for swift.common.wsgi"""
 
-    def setUp(self):
-        if six.PY2:
-            self._orig_parsetype = mimetools.Message.parsetype
-
-    def tearDown(self):
-        if six.PY2:
-            mimetools.Message.parsetype = self._orig_parsetype
-
-    @unittest.skipIf(six.PY3, "test specific to Python 2")
-    def test_monkey_patch_mimetools(self):
-        sio = StringIO('blah')
-        self.assertEqual(mimetools.Message(sio).type, 'text/plain')
-        sio = StringIO('blah')
-        self.assertEqual(mimetools.Message(sio).plisttext, '')
-        sio = StringIO('blah')
-        self.assertEqual(mimetools.Message(sio).maintype, 'text')
-        sio = StringIO('blah')
-        self.assertEqual(mimetools.Message(sio).subtype, 'plain')
-        sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
-        self.assertEqual(mimetools.Message(sio).type, 'text/html')
-        sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
-        self.assertEqual(mimetools.Message(sio).plisttext,
-                         '; charset=ISO-8859-4')
-        sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
-        self.assertEqual(mimetools.Message(sio).maintype, 'text')
-        sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
-        self.assertEqual(mimetools.Message(sio).subtype, 'html')
-
-        wsgi.monkey_patch_mimetools()
-        sio = StringIO('blah')
-        self.assertIsNone(mimetools.Message(sio).type)
-        sio = StringIO('blah')
-        self.assertEqual(mimetools.Message(sio).plisttext, '')
-        sio = StringIO('blah')
-        self.assertIsNone(mimetools.Message(sio).maintype)
-        sio = StringIO('blah')
-        self.assertIsNone(mimetools.Message(sio).subtype)
-        sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
-        self.assertEqual(mimetools.Message(sio).type, 'text/html')
-        sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
-        self.assertEqual(mimetools.Message(sio).plisttext,
-                         '; charset=ISO-8859-4')
-        sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
-        self.assertEqual(mimetools.Message(sio).maintype, 'text')
-        sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
-        self.assertEqual(mimetools.Message(sio).subtype, 'html')
-
     def test_init_request_processor(self):
         config = """
         [DEFAULT]

test/unit/helpers.py

@@ -40,6 +40,7 @@ from swift.common.storage_policy import StoragePolicy, ECStoragePolicy
 from swift.common.middleware import listing_formats, proxy_logging
 from swift.common import utils
 from swift.common.utils import mkdirs, normalize_timestamp, NullLogger
+from swift.common.wsgi import SwiftHttpProtocol
 from swift.container import server as container_server
 from swift.obj import server as object_server
 from swift.proxy import server as proxy_server
@@ -212,17 +213,28 @@ def setup_servers(the_object_server=object_server, extra_conf=None):
     nl = NullLogger()
     logging_prosv = proxy_logging.ProxyLoggingMiddleware(
         listing_formats.ListingFilter(prosrv), conf, logger=prosrv.logger)
-    prospa = spawn(wsgi.server, prolis, logging_prosv, nl)
+    prospa = spawn(wsgi.server, prolis, logging_prosv, nl,
-    acc1spa = spawn(wsgi.server, acc1lis, acc1srv, nl)
+                   protocol=SwiftHttpProtocol)
-    acc2spa = spawn(wsgi.server, acc2lis, acc2srv, nl)
+    acc1spa = spawn(wsgi.server, acc1lis, acc1srv, nl,
-    con1spa = spawn(wsgi.server, con1lis, con1srv, nl)
+                    protocol=SwiftHttpProtocol)
-    con2spa = spawn(wsgi.server, con2lis, con2srv, nl)
+    acc2spa = spawn(wsgi.server, acc2lis, acc2srv, nl,
-    obj1spa = spawn(wsgi.server, obj1lis, obj1srv, nl)
+                    protocol=SwiftHttpProtocol)
-    obj2spa = spawn(wsgi.server, obj2lis, obj2srv, nl)
+    con1spa = spawn(wsgi.server, con1lis, con1srv, nl,
-    obj3spa = spawn(wsgi.server, obj3lis, obj3srv, nl)
+                    protocol=SwiftHttpProtocol)
-    obj4spa = spawn(wsgi.server, obj4lis, obj4srv, nl)
+    con2spa = spawn(wsgi.server, con2lis, con2srv, nl,
-    obj5spa = spawn(wsgi.server, obj5lis, obj5srv, nl)
+                    protocol=SwiftHttpProtocol)
-    obj6spa = spawn(wsgi.server, obj6lis, obj6srv, nl)
+    obj1spa = spawn(wsgi.server, obj1lis, obj1srv, nl,
+                    protocol=SwiftHttpProtocol)
+    obj2spa = spawn(wsgi.server, obj2lis, obj2srv, nl,
+                    protocol=SwiftHttpProtocol)
+    obj3spa = spawn(wsgi.server, obj3lis, obj3srv, nl,
+                    protocol=SwiftHttpProtocol)
+    obj4spa = spawn(wsgi.server, obj4lis, obj4srv, nl,
+                    protocol=SwiftHttpProtocol)
+    obj5spa = spawn(wsgi.server, obj5lis, obj5srv, nl,
+                    protocol=SwiftHttpProtocol)
+    obj6spa = spawn(wsgi.server, obj6lis, obj6srv, nl,
+                    protocol=SwiftHttpProtocol)
     context["test_coros"] = \
         (prospa, acc1spa, acc2spa, con1spa, con2spa, obj1spa, obj2spa, obj3spa,
          obj4spa, obj5spa, obj6spa)

test/unit/proxy/test_server.py

@@ -71,7 +71,7 @@ from swift.common import utils, constraints
 from swift.common.utils import hash_path, storage_directory, \
     parse_content_type, parse_mime_headers, \
     iter_multipart_mime_documents, public, mkdirs, NullLogger
-from swift.common.wsgi import monkey_patch_mimetools, loadapp, ConfigString
+from swift.common.wsgi import loadapp, ConfigString
 from swift.proxy.controllers import base as proxy_base
 from swift.proxy.controllers.base import get_cache_key, cors_validation, \
     get_account_info, get_container_info
@@ -97,7 +97,6 @@ def do_setup(object_server):
     # setup test context and break out some globals for convenience
     global _test_context, _testdir, _test_servers, _test_sockets, \
         _test_POLICIES
-    monkey_patch_mimetools()
     _test_context = setup_servers(object_server)
     _testdir = _test_context["testdir"]
     _test_servers = _test_context["test_servers"]
@@ -3269,37 +3268,35 @@ class TestReplicatedObjectController(
         self.assertNotEqual(last_modified_put, last_modified_head)
         _do_conditional_GET_checks(last_modified_head)
 
+    @unpatch_policies
     def test_PUT_auto_content_type(self):
-        with save_globals():
+        prolis = _test_sockets[0]
-            controller = ReplicatedObjectController(
-                self.app, 'account', 'container', 'object')
 
-            def test_content_type(filename, expected):
+        def do_test(ext, content_type):
-                # The three responses here are for account_info() (HEAD to
+            sock = connect_tcp(('localhost', prolis.getsockname()[1]))
-                # account server), container_info() (HEAD to container server)
+            fd = sock.makefile('rwb')
-                # and three calls to _connect_put_node() (PUT to three object
+            fd.write(b'PUT /v1/a/c/o.%s HTTP/1.1\r\n'
-                # servers)
+                     b'Host: localhost\r\n'
-                set_http_connect(201, 201, 201, 201, 201,
+                     b'X-Storage-Token: t\r\nContent-Length: 0\r\n\r\n' %
-                                 give_content_type=lambda content_type:
+                     ext.encode())
-                                 self.assertEqual(content_type,
+            fd.flush()
-                                                  next(expected)))
+            headers = readuntil2crlfs(fd)
-                # We need into include a transfer-encoding to get past
+            exp = b'HTTP/1.1 201'
-                # constraints.check_object_creation()
+            self.assertEqual(headers[:len(exp)], exp)
-                req = Request.blank('/v1/a/c/%s' % filename, {},
-                                    headers={'transfer-encoding': 'chunked'})
-                self.app.update_request(req)
-                self.app.memcache.store = {}
-                res = controller.PUT(req)
-                # If we don't check the response here we could miss problems
-                # in PUT()
-                self.assertEqual(res.status_int, 201)
 
-            test_content_type('test.jpg', iter(['', '', 'image/jpeg',
+            fd.write(b'GET /v1/a/c/o.%s HTTP/1.1\r\n'
-                                                'image/jpeg', 'image/jpeg']))
+                     b'Host: localhost\r\nConnection: close\r\n'
-            test_content_type('test.html', iter(['', '', 'text/html',
+                     b'X-Storage-Token: t\r\n\r\n' % ext.encode())
-                                                 'text/html', 'text/html']))
+            fd.flush()
-            test_content_type('test.css', iter(['', '', 'text/css',
+            headers = readuntil2crlfs(fd)
-                                                'text/css', 'text/css']))
+            exp = b'HTTP/1.1 200'
+            self.assertIn(b'Content-Type: %s' % content_type.encode(),
+                          headers.split(b'\r\n'))
+            sock.close()
+
+        do_test('jpg', 'image/jpeg')
+        do_test('html', 'text/html')
+        do_test('css', 'text/css')
 
     def test_custom_mime_types_files(self):
         swift_dir = mkdtemp()

test/unit/proxy/test_sysmeta.py

@@ -24,7 +24,7 @@ from swift.common.middleware.copy import ServerSideCopyMiddleware
 from swift.common.storage_policy import StoragePolicy
 from swift.common.swob import Request
 from swift.common.utils import mkdirs, split_path
-from swift.common.wsgi import monkey_patch_mimetools, WSGIContext
+from swift.common.wsgi import WSGIContext
 from swift.obj import server as object_server
 from swift.proxy import server as proxy
 import swift.proxy.controllers
@@ -138,7 +138,6 @@ class TestObjectSysmeta(unittest.TestCase):
                                      account_ring=FakeRing(replicas=1),
                                      container_ring=FakeRing(replicas=1))
         self.copy_app = ServerSideCopyMiddleware(self.app, {})
-        monkey_patch_mimetools()
         self.tmpdir = mkdtemp()
         self.testdir = os.path.join(self.tmpdir,
                                     'tmp_test_object_server_ObjectController')