openstack/swift
Code Issues Proposed changes

Merge "Encode header in latin-1 with wsgi_to_bytes"

Browse Source
This commit is contained in:
Zuul 2023-07-10 17:44:30 +00:00 committed by Gerrit Code Review
commit 338908c830
6 changed files with 60 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
swift/obj/server.py
View File

@ -56,7 +56,7 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \
HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \ HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \
HTTPClientDisconnect, HTTPMethodNotAllowed, Request, Response, \ HTTPClientDisconnect, HTTPMethodNotAllowed, Request, Response, \
HTTPInsufficientStorage, HTTPForbidden, HTTPException, HTTPConflict, \ HTTPInsufficientStorage, HTTPForbidden, HTTPException, HTTPConflict, \
HTTPServerError, wsgi_to_bytes, wsgi_to_str, normalize_etag HTTPServerError, bytes_to_wsgi, wsgi_to_bytes, wsgi_to_str, normalize_etag
from swift.obj.diskfile import RESERVED_DATAFILE_META, DiskFileRouter from swift.obj.diskfile import RESERVED_DATAFILE_META, DiskFileRouter
from swift.obj.expirer import build_task_obj from swift.obj.expirer import build_task_obj
@ -678,7 +678,8 @@ class ObjectController(BaseStorageServer):
list(self.allowed_headers)) list(self.allowed_headers))
for header_key in headers_to_copy: for header_key in headers_to_copy:
if header_key in request.headers: if header_key in request.headers:
header_caps = header_key.title() header_caps = bytes_to_wsgi(
wsgi_to_bytes(header_key).title())
metadata[header_caps] = request.headers[header_key] metadata[header_caps] = request.headers[header_key]
orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0)
if orig_delete_at != new_delete_at: if orig_delete_at != new_delete_at:
@ -927,7 +928,8 @@ class ObjectController(BaseStorageServer):
list(self.allowed_headers)) list(self.allowed_headers))
for header_key in headers_to_copy: for header_key in headers_to_copy:
if header_key in request.headers: if header_key in request.headers:
header_caps = header_key.title() header_caps = bytes_to_wsgi(
wsgi_to_bytes(header_key).title())
metadata[header_caps] = request.headers[header_key] metadata[header_caps] = request.headers[header_key]
return metadata return metadata

6
swift/obj/ssync_receiver.py
View File

@ -476,9 +476,9 @@ class Receiver(object):
line = line.strip() line = line.strip()
if not line: if not line:
break break
header, value = swob.bytes_to_wsgi(line).split(':', 1) header, value = line.split(b':', 1)
header = header.strip().lower() header = swob.bytes_to_wsgi(header.strip().lower())
value = value.strip() value = swob.bytes_to_wsgi(value.strip())
subreq.headers[header] = value subreq.headers[header] = value
if header not in ('etag', 'x-backend-no-commit'): if header not in ('etag', 'x-backend-no-commit'):
# we'll use X-Backend-Replication-Headers to force the # we'll use X-Backend-Replication-Headers to force the

8
swift/obj/ssync_sender.py
View File

@ -21,6 +21,7 @@ from swift.common import bufferedhttp
from swift.common import exceptions from swift.common import exceptions
from swift.common import http from swift.common import http
from swift.common import utils from swift.common import utils
from swift.common.swob import wsgi_to_bytes
def encode_missing(object_hash, ts_data, ts_meta=None, ts_ctype=None, def encode_missing(object_hash, ts_data, ts_meta=None, ts_ctype=None,
@ -469,12 +470,7 @@ class Sender(object):
def send_subrequest(self, connection, method, url_path, headers, df): def send_subrequest(self, connection, method, url_path, headers, df):
msg = [b'%s %s' % (method.encode('ascii'), url_path.encode('utf8'))] msg = [b'%s %s' % (method.encode('ascii'), url_path.encode('utf8'))]
for key, value in sorted(headers.items()): for key, value in sorted(headers.items()):
if six.PY2: msg.append(wsgi_to_bytes('%s: %s' % (key, value)))
msg.append(b'%s: %s' % (key, value))
else:
msg.append(b'%s: %s' % (
key.encode('utf8', 'surrogateescape'),
str(value).encode('utf8', 'surrogateescape')))
msg = b'\r\n'.join(msg) + b'\r\n\r\n' msg = b'\r\n'.join(msg) + b'\r\n\r\n'
with exceptions.MessageTimeout(self.daemon.node_timeout, with exceptions.MessageTimeout(self.daemon.node_timeout,
'send_%s' % method.lower()): 'send_%s' % method.lower()):

8
test/unit/obj/test_server.py
View File

@ -273,6 +273,9 @@ class TestObjectController(BaseTestCase):
headers = {'X-Timestamp': post_timestamp, headers = {'X-Timestamp': post_timestamp,
'X-Object-Meta-3': 'Three', 'X-Object-Meta-3': 'Three',
'X-Object-Meta-4': 'Four', 'X-Object-Meta-4': 'Four',
'x-object-meta-t\xc3\xa8st': 'm\xc3\xa8ta',
'X-Backend-Replication-Headers':
'x-object-meta-t\xc3\xa8st',
'Content-Encoding': 'gzip', 'Content-Encoding': 'gzip',
'Foo': 'fooheader', 'Foo': 'fooheader',
'Bar': 'barheader'} 'Bar': 'barheader'}
@ -297,6 +300,7 @@ class TestObjectController(BaseTestCase):
'X-Object-Sysmeta-Color': 'blue', 'X-Object-Sysmeta-Color': 'blue',
'X-Object-Meta-3': 'Three', 'X-Object-Meta-3': 'Three',
'X-Object-Meta-4': 'Four', 'X-Object-Meta-4': 'Four',
'X-Object-Meta-T\xc3\xa8St': 'm\xc3\xa8ta',
'Foo': 'fooheader', 'Foo': 'fooheader',
'Bar': 'barheader', 'Bar': 'barheader',
'Content-Encoding': 'gzip', 'Content-Encoding': 'gzip',
@ -1424,9 +1428,10 @@ class TestObjectController(BaseTestCase):
'Content-Length': '6', 'Content-Length': '6',
'Content-Type': 'application/octet-stream', 'Content-Type': 'application/octet-stream',
'x-object-meta-test': 'one', 'x-object-meta-test': 'one',
'x-object-meta-t\xc3\xa8st': 'm\xc3\xa8ta',
'Custom-Header': '*', 'Custom-Header': '*',
'X-Backend-Replication-Headers': 'X-Backend-Replication-Headers':
'Content-Type Content-Length'}) 'x-object-meta-t\xc3\xa8st Content-Type Content-Length'})
req.body = 'VERIFY' req.body = 'VERIFY'
with mock.patch.object(self.object_controller, 'allowed_headers', with mock.patch.object(self.object_controller, 'allowed_headers',
['Custom-Header']): ['Custom-Header']):
@ -1448,6 +1453,7 @@ class TestObjectController(BaseTestCase):
'Content-Type': 'application/octet-stream', 'Content-Type': 'application/octet-stream',
'name': '/a/c/o', 'name': '/a/c/o',
'X-Object-Meta-Test': 'one', 'X-Object-Meta-Test': 'one',
'X-Object-Meta-T\xc3\xa8St': 'm\xc3\xa8ta',
'Custom-Header': '*'}) 'Custom-Header': '*'})
def test_PUT_overwrite(self): def test_PUT_overwrite(self):

32
test/unit/obj/test_ssync_receiver.py
View File

@ -1711,18 +1711,19 @@ class TestReceiver(unittest.TestCase):
req = swob.Request.blank( req = swob.Request.blank(
'/device/partition', '/device/partition',
environ={'REQUEST_METHOD': 'SSYNC'}, environ={'REQUEST_METHOD': 'SSYNC'},
body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' body=b':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n'
':UPDATES: START\r\n' b':UPDATES: START\r\n'
'PUT /a/c/o\r\n' b'PUT /a/c/o\r\n'
'Content-Length: 1\r\n' b'Content-Length: 1\r\n'
'Etag: c4ca4238a0b923820dcc509a6f75849b\r\n' b'Etag: c4ca4238a0b923820dcc509a6f75849b\r\n'
'X-Timestamp: 1364456113.12344\r\n' b'X-Timestamp: 1364456113.12344\r\n'
'X-Object-Meta-Test1: one\r\n' b'X-Object-Meta-Test1: one\r\n'
'Content-Encoding: gzip\r\n' b'X-Object-Meta-T\xc3\xa8st2: m\xc3\xa8ta\r\n'
'Specialty-Header: value\r\n' b'Content-Encoding: gzip\r\n'
'X-Backend-No-Commit: True\r\n' b'Specialty-Header: value\r\n'
'\r\n' b'X-Backend-No-Commit: True\r\n'
'1') b'\r\n'
b'1')
resp = req.get_response(self.controller) resp = req.get_response(self.controller)
self.assertEqual( self.assertEqual(
self.body_lines(resp.body), self.body_lines(resp.body),
@ -1735,11 +1736,12 @@ class TestReceiver(unittest.TestCase):
req = _PUT_request[0] req = _PUT_request[0]
self.assertEqual(req.path, '/device/partition/a/c/o') self.assertEqual(req.path, '/device/partition/a/c/o')
self.assertEqual(req.content_length, 1) self.assertEqual(req.content_length, 1)
self.assertEqual(req.headers, { expected = {
'Etag': 'c4ca4238a0b923820dcc509a6f75849b', 'Etag': 'c4ca4238a0b923820dcc509a6f75849b',
'Content-Length': '1', 'Content-Length': '1',
'X-Timestamp': '1364456113.12344', 'X-Timestamp': '1364456113.12344',
'X-Object-Meta-Test1': 'one', 'X-Object-Meta-Test1': 'one',
'X-Object-Meta-T\xc3\xa8st2': 'm\xc3\xa8ta',
'Content-Encoding': 'gzip', 'Content-Encoding': 'gzip',
'Specialty-Header': 'value', 'Specialty-Header': 'value',
'X-Backend-No-Commit': 'True', 'X-Backend-No-Commit': 'True',
@ -1749,7 +1751,9 @@ class TestReceiver(unittest.TestCase):
# note: Etag and X-Backend-No-Commit not in replication-headers # note: Etag and X-Backend-No-Commit not in replication-headers
'X-Backend-Replication-Headers': ( 'X-Backend-Replication-Headers': (
'content-length x-timestamp x-object-meta-test1 ' 'content-length x-timestamp x-object-meta-test1 '
'content-encoding specialty-header')}) 'x-object-meta-t\xc3\xa8st2 content-encoding '
'specialty-header')}
self.assertEqual({k: req.headers[k] for k in expected}, expected)
def test_UPDATES_PUT_replication_headers(self): def test_UPDATES_PUT_replication_headers(self):
self.controller.logger = mock.MagicMock() self.controller.logger = mock.MagicMock()

45
test/unit/obj/test_ssync_sender.py
View File

@ -23,6 +23,7 @@ import six
from swift.common import exceptions, utils from swift.common import exceptions, utils
from swift.common.storage_policy import POLICIES from swift.common.storage_policy import POLICIES
from swift.common.swob import wsgi_to_bytes, wsgi_to_str
from swift.common.utils import Timestamp from swift.common.utils import Timestamp
from swift.obj import ssync_sender, diskfile, ssync_receiver from swift.obj import ssync_sender, diskfile, ssync_receiver
from swift.obj.replicator import ObjectReplicator from swift.obj.replicator import ObjectReplicator
@ -1691,12 +1692,15 @@ class TestSender(BaseTest):
exc = err exc = err
self.assertEqual(str(exc), '0.01 seconds: send_put chunk') self.assertEqual(str(exc), '0.01 seconds: send_put chunk')
def _check_send_put(self, obj_name, meta_value, durable=True): def _check_send_put(self, obj_name, meta_value,
meta_name='Unicode-Meta-Name', durable=True):
ts_iter = make_timestamp_iter() ts_iter = make_timestamp_iter()
t1 = next(ts_iter) t1 = next(ts_iter)
body = b'test' body = b'test'
extra_metadata = {'Some-Other-Header': 'value', extra_metadata = {'Some-Other-Header': 'value',
u'Unicode-Meta-Name': meta_value} meta_name: meta_value}
# Note that diskfile expects obj_name to be a native string
# but metadata to be wsgi strings
df = self._make_open_diskfile(obj=obj_name, body=body, df = self._make_open_diskfile(obj=obj_name, body=body,
timestamp=t1, timestamp=t1,
extra_metadata=extra_metadata, extra_metadata=extra_metadata,
@ -1705,12 +1709,13 @@ class TestSender(BaseTest):
expected['body'] = body if six.PY2 else body.decode('ascii') expected['body'] = body if six.PY2 else body.decode('ascii')
expected['chunk_size'] = len(body) expected['chunk_size'] = len(body)
expected['meta'] = meta_value expected['meta'] = meta_value
wire_meta = meta_value if six.PY2 else meta_value.encode('utf8') expected['meta_name'] = meta_name
path = six.moves.urllib.parse.quote(expected['name']) path = six.moves.urllib.parse.quote(expected['name'])
expected['path'] = path expected['path'] = path
no_commit = '' if durable else 'X-Backend-No-Commit: True\r\n' no_commit = '' if durable else 'X-Backend-No-Commit: True\r\n'
expected['no_commit'] = no_commit expected['no_commit'] = no_commit
length = 145 + len(path) + len(wire_meta) + len(no_commit) length = 128 + len(path) + len(meta_value) + len(no_commit) + \
len(meta_name)
expected['length'] = format(length, 'x') expected['length'] = format(length, 'x')
# .meta file metadata is not included in expected for data only PUT # .meta file metadata is not included in expected for data only PUT
t2 = next(ts_iter) t2 = next(ts_iter)
@ -1725,15 +1730,14 @@ class TestSender(BaseTest):
'Content-Length: %(Content-Length)s\r\n' 'Content-Length: %(Content-Length)s\r\n'
'ETag: %(ETag)s\r\n' 'ETag: %(ETag)s\r\n'
'Some-Other-Header: value\r\n' 'Some-Other-Header: value\r\n'
'Unicode-Meta-Name: %(meta)s\r\n' '%(meta_name)s: %(meta)s\r\n'
'%(no_commit)s' '%(no_commit)s'
'X-Timestamp: %(X-Timestamp)s\r\n' 'X-Timestamp: %(X-Timestamp)s\r\n'
'\r\n' '\r\n'
'\r\n' '\r\n'
'%(chunk_size)s\r\n' '%(chunk_size)s\r\n'
'%(body)s\r\n' % expected) '%(body)s\r\n' % expected)
if not six.PY2: expected = wsgi_to_bytes(expected)
expected = expected.encode('utf8')
self.assertEqual(b''.join(connection.sent), expected) self.assertEqual(b''.join(connection.sent), expected)
def test_send_put(self): def test_send_put(self):
@ -1743,12 +1747,14 @@ class TestSender(BaseTest):
self._check_send_put('o', 'meta', durable=False) self._check_send_put('o', 'meta', durable=False)
def test_send_put_unicode(self): def test_send_put_unicode(self):
if six.PY2: self._check_send_put(
self._check_send_put( wsgi_to_str('o_with_caract\xc3\xa8res_like_in_french'),
'o_with_caract\xc3\xa8res_like_in_french', 'm\xc3\xa8ta') 'm\xc3\xa8ta')
else:
self._check_send_put( def test_send_put_unicode_header_name(self):
'o_with_caract\u00e8res_like_in_french', 'm\u00e8ta') self._check_send_put(
wsgi_to_str('o_with_caract\xc3\xa8res_like_in_french'),
'm\xc3\xa8ta', meta_name='X-Object-Meta-Nam\xc3\xa8')
def _check_send_post(self, obj_name, meta_value): def _check_send_post(self, obj_name, meta_value):
ts_iter = make_timestamp_iter() ts_iter = make_timestamp_iter()
@ -1764,9 +1770,11 @@ class TestSender(BaseTest):
ts_1 = next(ts_iter) ts_1 = next(ts_iter)
newer_metadata = {u'X-Object-Meta-Foo': meta_value, newer_metadata = {u'X-Object-Meta-Foo': meta_value,
'X-Timestamp': ts_1.internal} 'X-Timestamp': ts_1.internal}
# Note that diskfile expects obj_name to be a native string
# but metadata to be wsgi strings
df.write_metadata(newer_metadata) df.write_metadata(newer_metadata)
path = six.moves.urllib.parse.quote(df.read_metadata()['name']) path = six.moves.urllib.parse.quote(df.read_metadata()['name'])
wire_meta = meta_value if six.PY2 else meta_value.encode('utf8') wire_meta = wsgi_to_bytes(meta_value)
length = format(61 + len(path) + len(wire_meta), 'x') length = format(61 + len(path) + len(wire_meta), 'x')
connection = FakeConnection() connection = FakeConnection()
@ -1787,12 +1795,9 @@ class TestSender(BaseTest):
self._check_send_post('o', 'meta') self._check_send_post('o', 'meta')
def test_send_post_unicode(self): def test_send_post_unicode(self):
if six.PY2: self._check_send_post(
self._check_send_post( wsgi_to_str('o_with_caract\xc3\xa8res_like_in_french'),
'o_with_caract\xc3\xa8res_like_in_french', 'm\xc3\xa8ta') 'm\xc3\xa8ta')
else:
self._check_send_post(
'o_with_caract\u00e8res_like_in_french', 'm\u00e8ta')
def test_disconnect_timeout(self): def test_disconnect_timeout(self):
connection = FakeConnection() connection = FakeConnection()