openstack/swift
Code Issues Proposed changes

Encode header in latin-1 with wsgi_to_bytes

Browse Source 
Prevent encoding corruption in client's metadata during ssync

Closes-Bug: #2020667
Change-Id: I0ea464bcda16678997865667287aa11ea89cdcde
This commit is contained in:
Romain de Joux 2023-05-23 11:15:14 +02:00 committed by Alistair Coles
parent ca3f107706
commit 365c0ef005
6 changed files with 60 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
swift/obj/server.py
View File

@ -56,7 +56,7 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \
HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \
HTTPClientDisconnect, HTTPMethodNotAllowed, Request, Response, \
HTTPInsufficientStorage, HTTPForbidden, HTTPException, HTTPConflict, \
HTTPServerError, wsgi_to_bytes, wsgi_to_str, normalize_etag
HTTPServerError, bytes_to_wsgi, wsgi_to_bytes, wsgi_to_str, normalize_etag
from swift.obj.diskfile import RESERVED_DATAFILE_META, DiskFileRouter
from swift.obj.expirer import build_task_obj
@ -678,7 +678,8 @@ class ObjectController(BaseStorageServer):
list(self.allowed_headers))
for header_key in headers_to_copy:
if header_key in request.headers:
header_caps = header_key.title()
header_caps = bytes_to_wsgi(
wsgi_to_bytes(header_key).title())
metadata[header_caps] = request.headers[header_key]
orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0)
if orig_delete_at != new_delete_at:
@ -927,7 +928,8 @@ class ObjectController(BaseStorageServer):
list(self.allowed_headers))
for header_key in headers_to_copy:
if header_key in request.headers:
header_caps = header_key.title()
header_caps = bytes_to_wsgi(
wsgi_to_bytes(header_key).title())
metadata[header_caps] = request.headers[header_key]
return metadata

6
swift/obj/ssync_receiver.py
View File

@ -476,9 +476,9 @@ class Receiver(object):
line = line.strip()
if not line:
break
header, value = swob.bytes_to_wsgi(line).split(':', 1)
header = header.strip().lower()
value = value.strip()
header, value = line.split(b':', 1)
header = swob.bytes_to_wsgi(header.strip().lower())
value = swob.bytes_to_wsgi(value.strip())
subreq.headers[header] = value
if header not in ('etag', 'x-backend-no-commit'):
# we'll use X-Backend-Replication-Headers to force the

8
swift/obj/ssync_sender.py
View File

@ -21,6 +21,7 @@ from swift.common import bufferedhttp
from swift.common import exceptions
from swift.common import http
from swift.common import utils
from swift.common.swob import wsgi_to_bytes
def encode_missing(object_hash, ts_data, ts_meta=None, ts_ctype=None,
@ -469,12 +470,7 @@ class Sender(object):
def send_subrequest(self, connection, method, url_path, headers, df):
msg = [b'%s %s' % (method.encode('ascii'), url_path.encode('utf8'))]
for key, value in sorted(headers.items()):
if six.PY2:
msg.append(b'%s: %s' % (key, value))
else:
msg.append(b'%s: %s' % (
key.encode('utf8', 'surrogateescape'),
str(value).encode('utf8', 'surrogateescape')))
msg.append(wsgi_to_bytes('%s: %s' % (key, value)))
msg = b'\r\n'.join(msg) + b'\r\n\r\n'
with exceptions.MessageTimeout(self.daemon.node_timeout,
'send_%s' % method.lower()):

8
test/unit/obj/test_server.py
View File

@ -273,6 +273,9 @@ class TestObjectController(BaseTestCase):
headers = {'X-Timestamp': post_timestamp,
'X-Object-Meta-3': 'Three',
'X-Object-Meta-4': 'Four',
'x-object-meta-t\xc3\xa8st': 'm\xc3\xa8ta',
'X-Backend-Replication-Headers':
'x-object-meta-t\xc3\xa8st',
'Content-Encoding': 'gzip',
'Foo': 'fooheader',
'Bar': 'barheader'}
@ -297,6 +300,7 @@ class TestObjectController(BaseTestCase):
'X-Object-Sysmeta-Color': 'blue',
'X-Object-Meta-3': 'Three',
'X-Object-Meta-4': 'Four',
'X-Object-Meta-T\xc3\xa8St': 'm\xc3\xa8ta',
'Foo': 'fooheader',
'Bar': 'barheader',
'Content-Encoding': 'gzip',
@ -1424,9 +1428,10 @@ class TestObjectController(BaseTestCase):
'Content-Length': '6',
'Content-Type': 'application/octet-stream',
'x-object-meta-test': 'one',
'x-object-meta-t\xc3\xa8st': 'm\xc3\xa8ta',
'Custom-Header': '*',
'X-Backend-Replication-Headers':
'Content-Type Content-Length'})
'x-object-meta-t\xc3\xa8st Content-Type Content-Length'})
req.body = 'VERIFY'
with mock.patch.object(self.object_controller, 'allowed_headers',
['Custom-Header']):
@ -1448,6 +1453,7 @@ class TestObjectController(BaseTestCase):
'Content-Type': 'application/octet-stream',
'name': '/a/c/o',
'X-Object-Meta-Test': 'one',
'X-Object-Meta-T\xc3\xa8St': 'm\xc3\xa8ta',
'Custom-Header': '*'})
def test_PUT_overwrite(self):

32
test/unit/obj/test_ssync_receiver.py
View File

@ -1711,18 +1711,19 @@ class TestReceiver(unittest.TestCase):
req = swob.Request.blank(
'/device/partition',
environ={'REQUEST_METHOD': 'SSYNC'},
body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n'
':UPDATES: START\r\n'
'PUT /a/c/o\r\n'
'Content-Length: 1\r\n'
'Etag: c4ca4238a0b923820dcc509a6f75849b\r\n'
'X-Timestamp: 1364456113.12344\r\n'
'X-Object-Meta-Test1: one\r\n'
'Content-Encoding: gzip\r\n'
'Specialty-Header: value\r\n'
'X-Backend-No-Commit: True\r\n'
'\r\n'
'1')
body=b':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n'
b':UPDATES: START\r\n'
b'PUT /a/c/o\r\n'
b'Content-Length: 1\r\n'
b'Etag: c4ca4238a0b923820dcc509a6f75849b\r\n'
b'X-Timestamp: 1364456113.12344\r\n'
b'X-Object-Meta-Test1: one\r\n'
b'X-Object-Meta-T\xc3\xa8st2: m\xc3\xa8ta\r\n'
b'Content-Encoding: gzip\r\n'
b'Specialty-Header: value\r\n'
b'X-Backend-No-Commit: True\r\n'
b'\r\n'
b'1')
resp = req.get_response(self.controller)
self.assertEqual(
self.body_lines(resp.body),
@ -1735,11 +1736,12 @@ class TestReceiver(unittest.TestCase):
req = _PUT_request[0]
self.assertEqual(req.path, '/device/partition/a/c/o')
self.assertEqual(req.content_length, 1)
self.assertEqual(req.headers, {
expected = {
'Etag': 'c4ca4238a0b923820dcc509a6f75849b',
'Content-Length': '1',
'X-Timestamp': '1364456113.12344',
'X-Object-Meta-Test1': 'one',
'X-Object-Meta-T\xc3\xa8st2': 'm\xc3\xa8ta',
'Content-Encoding': 'gzip',
'Specialty-Header': 'value',
'X-Backend-No-Commit': 'True',
@ -1749,7 +1751,9 @@ class TestReceiver(unittest.TestCase):
# note: Etag and X-Backend-No-Commit not in replication-headers
'X-Backend-Replication-Headers': (
'content-length x-timestamp x-object-meta-test1 '
'content-encoding specialty-header')})
'x-object-meta-t\xc3\xa8st2 content-encoding '
'specialty-header')}
self.assertEqual({k: req.headers[k] for k in expected}, expected)
def test_UPDATES_PUT_replication_headers(self):
self.controller.logger = mock.MagicMock()

45
test/unit/obj/test_ssync_sender.py
View File

@ -23,6 +23,7 @@ import six
from swift.common import exceptions, utils
from swift.common.storage_policy import POLICIES
from swift.common.swob import wsgi_to_bytes, wsgi_to_str
from swift.common.utils import Timestamp
from swift.obj import ssync_sender, diskfile, ssync_receiver
from swift.obj.replicator import ObjectReplicator
@ -1691,12 +1692,15 @@ class TestSender(BaseTest):
exc = err
self.assertEqual(str(exc), '0.01 seconds: send_put chunk')
def _check_send_put(self, obj_name, meta_value, durable=True):
def _check_send_put(self, obj_name, meta_value,
meta_name='Unicode-Meta-Name', durable=True):
ts_iter = make_timestamp_iter()
t1 = next(ts_iter)
body = b'test'
extra_metadata = {'Some-Other-Header': 'value',
u'Unicode-Meta-Name': meta_value}
meta_name: meta_value}
# Note that diskfile expects obj_name to be a native string
# but metadata to be wsgi strings
df = self._make_open_diskfile(obj=obj_name, body=body,
timestamp=t1,
extra_metadata=extra_metadata,
@ -1705,12 +1709,13 @@ class TestSender(BaseTest):
expected['body'] = body if six.PY2 else body.decode('ascii')
expected['chunk_size'] = len(body)
expected['meta'] = meta_value
wire_meta = meta_value if six.PY2 else meta_value.encode('utf8')
expected['meta_name'] = meta_name
path = six.moves.urllib.parse.quote(expected['name'])
expected['path'] = path
no_commit = '' if durable else 'X-Backend-No-Commit: True\r\n'
expected['no_commit'] = no_commit
length = 145 + len(path) + len(wire_meta) + len(no_commit)
length = 128 + len(path) + len(meta_value) + len(no_commit) + \
len(meta_name)
expected['length'] = format(length, 'x')
# .meta file metadata is not included in expected for data only PUT
t2 = next(ts_iter)
@ -1725,15 +1730,14 @@ class TestSender(BaseTest):
'Content-Length: %(Content-Length)s\r\n'
'ETag: %(ETag)s\r\n'
'Some-Other-Header: value\r\n'
'Unicode-Meta-Name: %(meta)s\r\n'
'%(meta_name)s: %(meta)s\r\n'
'%(no_commit)s'
'X-Timestamp: %(X-Timestamp)s\r\n'
'\r\n'
'\r\n'
'%(chunk_size)s\r\n'
'%(body)s\r\n' % expected)
if not six.PY2:
expected = expected.encode('utf8')
expected = wsgi_to_bytes(expected)
self.assertEqual(b''.join(connection.sent), expected)
def test_send_put(self):
@ -1743,12 +1747,14 @@ class TestSender(BaseTest):
self._check_send_put('o', 'meta', durable=False)
def test_send_put_unicode(self):
if six.PY2:
self._check_send_put(
'o_with_caract\xc3\xa8res_like_in_french', 'm\xc3\xa8ta')
else:
self._check_send_put(
'o_with_caract\u00e8res_like_in_french', 'm\u00e8ta')
self._check_send_put(
wsgi_to_str('o_with_caract\xc3\xa8res_like_in_french'),
'm\xc3\xa8ta')
def test_send_put_unicode_header_name(self):
self._check_send_put(
wsgi_to_str('o_with_caract\xc3\xa8res_like_in_french'),
'm\xc3\xa8ta', meta_name='X-Object-Meta-Nam\xc3\xa8')
def _check_send_post(self, obj_name, meta_value):
ts_iter = make_timestamp_iter()
@ -1764,9 +1770,11 @@ class TestSender(BaseTest):
ts_1 = next(ts_iter)
newer_metadata = {u'X-Object-Meta-Foo': meta_value,
'X-Timestamp': ts_1.internal}
# Note that diskfile expects obj_name to be a native string
# but metadata to be wsgi strings
df.write_metadata(newer_metadata)
path = six.moves.urllib.parse.quote(df.read_metadata()['name'])
wire_meta = meta_value if six.PY2 else meta_value.encode('utf8')
wire_meta = wsgi_to_bytes(meta_value)
length = format(61 + len(path) + len(wire_meta), 'x')
connection = FakeConnection()
@ -1787,12 +1795,9 @@ class TestSender(BaseTest):
self._check_send_post('o', 'meta')
def test_send_post_unicode(self):
if six.PY2:
self._check_send_post(
'o_with_caract\xc3\xa8res_like_in_french', 'm\xc3\xa8ta')
else:
self._check_send_post(
'o_with_caract\u00e8res_like_in_french', 'm\u00e8ta')
self._check_send_post(
wsgi_to_str('o_with_caract\xc3\xa8res_like_in_french'),
'm\xc3\xa8ta')
def test_disconnect_timeout(self):
connection = FakeConnection()