diff --git a/swift/obj/server.py b/swift/obj/server.py index e157323bb3..0e47fd8188 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -56,7 +56,7 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \ HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \ HTTPClientDisconnect, HTTPMethodNotAllowed, Request, Response, \ HTTPInsufficientStorage, HTTPForbidden, HTTPException, HTTPConflict, \ - HTTPServerError, wsgi_to_bytes, wsgi_to_str, normalize_etag + HTTPServerError, bytes_to_wsgi, wsgi_to_bytes, wsgi_to_str, normalize_etag from swift.obj.diskfile import RESERVED_DATAFILE_META, DiskFileRouter from swift.obj.expirer import build_task_obj @@ -678,7 +678,8 @@ class ObjectController(BaseStorageServer): list(self.allowed_headers)) for header_key in headers_to_copy: if header_key in request.headers: - header_caps = header_key.title() + header_caps = bytes_to_wsgi( + wsgi_to_bytes(header_key).title()) metadata[header_caps] = request.headers[header_key] orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) if orig_delete_at != new_delete_at: @@ -927,7 +928,8 @@ class ObjectController(BaseStorageServer): list(self.allowed_headers)) for header_key in headers_to_copy: if header_key in request.headers: - header_caps = header_key.title() + header_caps = bytes_to_wsgi( + wsgi_to_bytes(header_key).title()) metadata[header_caps] = request.headers[header_key] return metadata diff --git a/swift/obj/ssync_receiver.py b/swift/obj/ssync_receiver.py index fb125fca2d..de75eaa9d4 100644 --- a/swift/obj/ssync_receiver.py +++ b/swift/obj/ssync_receiver.py @@ -476,9 +476,9 @@ class Receiver(object): line = line.strip() if not line: break - header, value = swob.bytes_to_wsgi(line).split(':', 1) - header = header.strip().lower() - value = value.strip() + header, value = line.split(b':', 1) + header = swob.bytes_to_wsgi(header.strip().lower()) + value = swob.bytes_to_wsgi(value.strip()) subreq.headers[header] = value if header not in ('etag', 'x-backend-no-commit'): # we'll use X-Backend-Replication-Headers to force the diff --git a/swift/obj/ssync_sender.py b/swift/obj/ssync_sender.py index b132b8b3df..2ebfbd8512 100644 --- a/swift/obj/ssync_sender.py +++ b/swift/obj/ssync_sender.py @@ -21,6 +21,7 @@ from swift.common import bufferedhttp from swift.common import exceptions from swift.common import http from swift.common import utils +from swift.common.swob import wsgi_to_bytes def encode_missing(object_hash, ts_data, ts_meta=None, ts_ctype=None, @@ -469,12 +470,7 @@ class Sender(object): def send_subrequest(self, connection, method, url_path, headers, df): msg = [b'%s %s' % (method.encode('ascii'), url_path.encode('utf8'))] for key, value in sorted(headers.items()): - if six.PY2: - msg.append(b'%s: %s' % (key, value)) - else: - msg.append(b'%s: %s' % ( - key.encode('utf8', 'surrogateescape'), - str(value).encode('utf8', 'surrogateescape'))) + msg.append(wsgi_to_bytes('%s: %s' % (key, value))) msg = b'\r\n'.join(msg) + b'\r\n\r\n' with exceptions.MessageTimeout(self.daemon.node_timeout, 'send_%s' % method.lower()): diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index 33405267f9..be7e2f2def 100644 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -273,6 +273,9 @@ class TestObjectController(BaseTestCase): headers = {'X-Timestamp': post_timestamp, 'X-Object-Meta-3': 'Three', 'X-Object-Meta-4': 'Four', + 'x-object-meta-t\xc3\xa8st': 'm\xc3\xa8ta', + 'X-Backend-Replication-Headers': + 'x-object-meta-t\xc3\xa8st', 'Content-Encoding': 'gzip', 'Foo': 'fooheader', 'Bar': 'barheader'} @@ -297,6 +300,7 @@ class TestObjectController(BaseTestCase): 'X-Object-Sysmeta-Color': 'blue', 'X-Object-Meta-3': 'Three', 'X-Object-Meta-4': 'Four', + 'X-Object-Meta-T\xc3\xa8St': 'm\xc3\xa8ta', 'Foo': 'fooheader', 'Bar': 'barheader', 'Content-Encoding': 'gzip', @@ -1424,9 +1428,10 @@ class TestObjectController(BaseTestCase): 'Content-Length': '6', 'Content-Type': 'application/octet-stream', 'x-object-meta-test': 'one', + 'x-object-meta-t\xc3\xa8st': 'm\xc3\xa8ta', 'Custom-Header': '*', 'X-Backend-Replication-Headers': - 'Content-Type Content-Length'}) + 'x-object-meta-t\xc3\xa8st Content-Type Content-Length'}) req.body = 'VERIFY' with mock.patch.object(self.object_controller, 'allowed_headers', ['Custom-Header']): @@ -1448,6 +1453,7 @@ class TestObjectController(BaseTestCase): 'Content-Type': 'application/octet-stream', 'name': '/a/c/o', 'X-Object-Meta-Test': 'one', + 'X-Object-Meta-T\xc3\xa8St': 'm\xc3\xa8ta', 'Custom-Header': '*'}) def test_PUT_overwrite(self): diff --git a/test/unit/obj/test_ssync_receiver.py b/test/unit/obj/test_ssync_receiver.py index 7093927449..74ae43064c 100644 --- a/test/unit/obj/test_ssync_receiver.py +++ b/test/unit/obj/test_ssync_receiver.py @@ -1711,18 +1711,19 @@ class TestReceiver(unittest.TestCase): req = swob.Request.blank( '/device/partition', environ={'REQUEST_METHOD': 'SSYNC'}, - body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' - ':UPDATES: START\r\n' - 'PUT /a/c/o\r\n' - 'Content-Length: 1\r\n' - 'Etag: c4ca4238a0b923820dcc509a6f75849b\r\n' - 'X-Timestamp: 1364456113.12344\r\n' - 'X-Object-Meta-Test1: one\r\n' - 'Content-Encoding: gzip\r\n' - 'Specialty-Header: value\r\n' - 'X-Backend-No-Commit: True\r\n' - '\r\n' - '1') + body=b':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' + b':UPDATES: START\r\n' + b'PUT /a/c/o\r\n' + b'Content-Length: 1\r\n' + b'Etag: c4ca4238a0b923820dcc509a6f75849b\r\n' + b'X-Timestamp: 1364456113.12344\r\n' + b'X-Object-Meta-Test1: one\r\n' + b'X-Object-Meta-T\xc3\xa8st2: m\xc3\xa8ta\r\n' + b'Content-Encoding: gzip\r\n' + b'Specialty-Header: value\r\n' + b'X-Backend-No-Commit: True\r\n' + b'\r\n' + b'1') resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), @@ -1735,11 +1736,12 @@ class TestReceiver(unittest.TestCase): req = _PUT_request[0] self.assertEqual(req.path, '/device/partition/a/c/o') self.assertEqual(req.content_length, 1) - self.assertEqual(req.headers, { + expected = { 'Etag': 'c4ca4238a0b923820dcc509a6f75849b', 'Content-Length': '1', 'X-Timestamp': '1364456113.12344', 'X-Object-Meta-Test1': 'one', + 'X-Object-Meta-T\xc3\xa8st2': 'm\xc3\xa8ta', 'Content-Encoding': 'gzip', 'Specialty-Header': 'value', 'X-Backend-No-Commit': 'True', @@ -1749,7 +1751,9 @@ class TestReceiver(unittest.TestCase): # note: Etag and X-Backend-No-Commit not in replication-headers 'X-Backend-Replication-Headers': ( 'content-length x-timestamp x-object-meta-test1 ' - 'content-encoding specialty-header')}) + 'x-object-meta-t\xc3\xa8st2 content-encoding ' + 'specialty-header')} + self.assertEqual({k: req.headers[k] for k in expected}, expected) def test_UPDATES_PUT_replication_headers(self): self.controller.logger = mock.MagicMock() diff --git a/test/unit/obj/test_ssync_sender.py b/test/unit/obj/test_ssync_sender.py index d577f6867e..89dcd3211d 100644 --- a/test/unit/obj/test_ssync_sender.py +++ b/test/unit/obj/test_ssync_sender.py @@ -23,6 +23,7 @@ import six from swift.common import exceptions, utils from swift.common.storage_policy import POLICIES +from swift.common.swob import wsgi_to_bytes, wsgi_to_str from swift.common.utils import Timestamp from swift.obj import ssync_sender, diskfile, ssync_receiver from swift.obj.replicator import ObjectReplicator @@ -1691,12 +1692,15 @@ class TestSender(BaseTest): exc = err self.assertEqual(str(exc), '0.01 seconds: send_put chunk') - def _check_send_put(self, obj_name, meta_value, durable=True): + def _check_send_put(self, obj_name, meta_value, + meta_name='Unicode-Meta-Name', durable=True): ts_iter = make_timestamp_iter() t1 = next(ts_iter) body = b'test' extra_metadata = {'Some-Other-Header': 'value', - u'Unicode-Meta-Name': meta_value} + meta_name: meta_value} + # Note that diskfile expects obj_name to be a native string + # but metadata to be wsgi strings df = self._make_open_diskfile(obj=obj_name, body=body, timestamp=t1, extra_metadata=extra_metadata, @@ -1705,12 +1709,13 @@ class TestSender(BaseTest): expected['body'] = body if six.PY2 else body.decode('ascii') expected['chunk_size'] = len(body) expected['meta'] = meta_value - wire_meta = meta_value if six.PY2 else meta_value.encode('utf8') + expected['meta_name'] = meta_name path = six.moves.urllib.parse.quote(expected['name']) expected['path'] = path no_commit = '' if durable else 'X-Backend-No-Commit: True\r\n' expected['no_commit'] = no_commit - length = 145 + len(path) + len(wire_meta) + len(no_commit) + length = 128 + len(path) + len(meta_value) + len(no_commit) + \ + len(meta_name) expected['length'] = format(length, 'x') # .meta file metadata is not included in expected for data only PUT t2 = next(ts_iter) @@ -1725,15 +1730,14 @@ class TestSender(BaseTest): 'Content-Length: %(Content-Length)s\r\n' 'ETag: %(ETag)s\r\n' 'Some-Other-Header: value\r\n' - 'Unicode-Meta-Name: %(meta)s\r\n' + '%(meta_name)s: %(meta)s\r\n' '%(no_commit)s' 'X-Timestamp: %(X-Timestamp)s\r\n' '\r\n' '\r\n' '%(chunk_size)s\r\n' '%(body)s\r\n' % expected) - if not six.PY2: - expected = expected.encode('utf8') + expected = wsgi_to_bytes(expected) self.assertEqual(b''.join(connection.sent), expected) def test_send_put(self): @@ -1743,12 +1747,14 @@ class TestSender(BaseTest): self._check_send_put('o', 'meta', durable=False) def test_send_put_unicode(self): - if six.PY2: - self._check_send_put( - 'o_with_caract\xc3\xa8res_like_in_french', 'm\xc3\xa8ta') - else: - self._check_send_put( - 'o_with_caract\u00e8res_like_in_french', 'm\u00e8ta') + self._check_send_put( + wsgi_to_str('o_with_caract\xc3\xa8res_like_in_french'), + 'm\xc3\xa8ta') + + def test_send_put_unicode_header_name(self): + self._check_send_put( + wsgi_to_str('o_with_caract\xc3\xa8res_like_in_french'), + 'm\xc3\xa8ta', meta_name='X-Object-Meta-Nam\xc3\xa8') def _check_send_post(self, obj_name, meta_value): ts_iter = make_timestamp_iter() @@ -1764,9 +1770,11 @@ class TestSender(BaseTest): ts_1 = next(ts_iter) newer_metadata = {u'X-Object-Meta-Foo': meta_value, 'X-Timestamp': ts_1.internal} + # Note that diskfile expects obj_name to be a native string + # but metadata to be wsgi strings df.write_metadata(newer_metadata) path = six.moves.urllib.parse.quote(df.read_metadata()['name']) - wire_meta = meta_value if six.PY2 else meta_value.encode('utf8') + wire_meta = wsgi_to_bytes(meta_value) length = format(61 + len(path) + len(wire_meta), 'x') connection = FakeConnection() @@ -1787,12 +1795,9 @@ class TestSender(BaseTest): self._check_send_post('o', 'meta') def test_send_post_unicode(self): - if six.PY2: - self._check_send_post( - 'o_with_caract\xc3\xa8res_like_in_french', 'm\xc3\xa8ta') - else: - self._check_send_post( - 'o_with_caract\u00e8res_like_in_french', 'm\u00e8ta') + self._check_send_post( + wsgi_to_str('o_with_caract\xc3\xa8res_like_in_french'), + 'm\xc3\xa8ta') def test_disconnect_timeout(self): connection = FakeConnection()