From ef664700d42db5bd2dd8be073598eee36ab1b258 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Fri, 14 Jun 2019 15:41:13 -0700 Subject: [PATCH] py3: Specify an encoding when loading db.pending pickles Otherwise you can hit UnicodeDecodeErrors when upgrading from py2 to py3. Change-Id: Iab808b3d0c2051badda0450015077ea8b1e5adb8 --- swift/common/db.py | 6 +++++- test/unit/common/test_db.py | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/swift/common/db.py b/swift/common/db.py index 0ce106fcc5..2fdc94260e 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -730,7 +730,11 @@ class DatabaseBroker(object): for entry in fp.read().split(b':'): if entry: try: - data = pickle.loads(base64.b64decode(entry)) + if six.PY2: + data = pickle.loads(base64.b64decode(entry)) + else: + data = pickle.loads(base64.b64decode(entry), + encoding='utf8') self._commit_puts_load(item_list, data) except Exception: self.logger.exception( diff --git a/test/unit/common/test_db.py b/test/unit/common/test_db.py index ebf240e724..1226e0f0dc 100644 --- a/test/unit/common/test_db.py +++ b/test/unit/common/test_db.py @@ -1441,6 +1441,23 @@ class TestDatabaseBroker(unittest.TestCase): mock_merge_items.assert_called_once_with([b'not', b'bad']) self.assertEqual(0, os.path.getsize(broker.pending_file)) + # load a pending entry that's caused trouble in py2/py3 upgrade tests + # can't quite figure out how it got generated, though, so hard-code it + with open(broker.pending_file, 'wb') as fd: + fd.write(b':gAIoVS3olIngpILrjIvrjIvpkIngpIHlmIjlmIbjnIbgp' + b'IPjnITimIPvhI/rjI3tiI5xAVUQMTU1OTI0MTg0Ni40NjY' + b'wMXECVQEwVQEwVQEwSwBVATB0Lg==') + with patch.object(broker, 'merge_items') as mock_merge_items: + broker._commit_puts_load = lambda l, e: l.append(e) + broker._commit_puts([]) + expected_name = (u'\u8509\u0902\ub30b\ub30b\u9409\u0901\u5608\u5606' + u'\u3706\u0903\u3704\u2603\uf10f\ub30d\ud20e') + if six.PY2: + expected_name = expected_name.encode('utf8') + mock_merge_items.assert_called_once_with([ + (expected_name, '1559241846.46601', '0', '0', '0', 0, '0')]) + self.assertEqual(0, os.path.getsize(broker.pending_file)) + # skip_commits True - no merge db_file = os.path.join(self.testdir, '2.db') broker = DatabaseBroker(db_file, skip_commits=True)