openstack/swift
Code Issues Proposed changes

py3: Specify an encoding when loading db.pending pickles

Browse Source 
Otherwise you can hit UnicodeDecodeErrors when upgrading from py2 to
py3.

Change-Id: Iab808b3d0c2051badda0450015077ea8b1e5adb8
This commit is contained in:
Tim Burke 2019-06-14 15:41:13 -07:00
parent 74e1f2e053
commit ef664700d4
2 changed files with 22 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
swift/common/db.py
View File

@ -730,7 +730,11 @@ class DatabaseBroker(object):
for entry in fp.read().split(b':'):
if entry:
try:
data = pickle.loads(base64.b64decode(entry))
if six.PY2:
data = pickle.loads(base64.b64decode(entry))
else:
data = pickle.loads(base64.b64decode(entry),
encoding='utf8')
self._commit_puts_load(item_list, data)
except Exception:
self.logger.exception(

17
test/unit/common/test_db.py
View File

@ -1441,6 +1441,23 @@ class TestDatabaseBroker(unittest.TestCase):
mock_merge_items.assert_called_once_with([b'not', b'bad'])
self.assertEqual(0, os.path.getsize(broker.pending_file))
# load a pending entry that's caused trouble in py2/py3 upgrade tests
# can't quite figure out how it got generated, though, so hard-code it
with open(broker.pending_file, 'wb') as fd:
fd.write(b':gAIoVS3olIngpILrjIvrjIvpkIngpIHlmIjlmIbjnIbgp'
b'IPjnITimIPvhI/rjI3tiI5xAVUQMTU1OTI0MTg0Ni40NjY'
b'wMXECVQEwVQEwVQEwSwBVATB0Lg==')
with patch.object(broker, 'merge_items') as mock_merge_items:
broker._commit_puts_load = lambda l, e: l.append(e)
broker._commit_puts([])
expected_name = (u'\u8509\u0902\ub30b\ub30b\u9409\u0901\u5608\u5606'
u'\u3706\u0903\u3704\u2603\uf10f\ub30d\ud20e')
if six.PY2:
expected_name = expected_name.encode('utf8')
mock_merge_items.assert_called_once_with([
(expected_name, '1559241846.46601', '0', '0', '0', 0, '0')])
self.assertEqual(0, os.path.getsize(broker.pending_file))
# skip_commits True - no merge
db_file = os.path.join(self.testdir, '2.db')
broker = DatabaseBroker(db_file, skip_commits=True)