diff --git a/swift/obj/diskfile.py b/swift/obj/diskfile.py index 9e69e954f2..620e641086 100644 --- a/swift/obj/diskfile.py +++ b/swift/obj/diskfile.py @@ -49,6 +49,7 @@ from collections import defaultdict from eventlet import Timeout from eventlet.hubs import trampoline +import six from swift import gettext_ as _ from swift.common.constraints import check_mount, check_dir @@ -112,7 +113,7 @@ def read_metadata(fd): :returns: dictionary of metadata """ - metadata = '' + metadata = b'' key = 0 try: while True: @@ -281,7 +282,7 @@ def consolidate_hashes(partition_dir): # Now that all the invalidations are reflected in hashes.pkl, it's # safe to clear out the invalidations file. try: - with open(invalidations_file, 'w') as inv_fh: + with open(invalidations_file, 'wb') as inv_fh: pass except OSError as e: if e.errno != errno.ENOENT: @@ -416,7 +417,11 @@ def get_auditor_status(datadir_path, logger, auditor_type): datadir_path, "auditor_status_%s.json" % auditor_type) status = {} try: - with open(auditor_status) as statusfile: + if six.PY3: + statusfile = open(auditor_status, encoding='utf8') + else: + statusfile = open(auditor_status, 'rb') + with statusfile: status = statusfile.read() except (OSError, IOError) as e: if e.errno != errno.ENOENT and logger: @@ -435,6 +440,8 @@ def get_auditor_status(datadir_path, logger, auditor_type): def update_auditor_status(datadir_path, logger, partitions, auditor_type): status = json.dumps({'partitions': partitions}) + if six.PY3: + status = status.encode('utf8') auditor_status = os.path.join( datadir_path, "auditor_status_%s.json" % auditor_type) try: @@ -2516,7 +2523,7 @@ class ECDiskFileWriter(BaseDiskFileWriter): exc = None try: try: - with open(durable_file_path, 'w') as _fp: + with open(durable_file_path, 'wb') as _fp: fsync(_fp.fileno()) fsync_dir(self._datadir) except (OSError, IOError) as err: diff --git a/test/unit/obj/test_auditor.py b/test/unit/obj/test_auditor.py index 78aa08a246..afc3657431 100644 --- a/test/unit/obj/test_auditor.py +++ b/test/unit/obj/test_auditor.py @@ -153,7 +153,7 @@ class TestAuditor(unittest.TestCase): def run_tests(disk_file): auditor_worker = auditor.AuditorWorker(self.conf, self.logger, self.rcache, self.devices) - data = '0' * 1024 + data = b'0' * 1024 etag = md5() with disk_file.create() as writer: writer.write(data) @@ -174,7 +174,7 @@ class TestAuditor(unittest.TestCase): policy=disk_file.policy)) self.assertEqual(auditor_worker.quarantines, pre_quarantines) - os.write(writer._fd, 'extra_data') + os.write(writer._fd, b'extra_data') auditor_worker.object_audit( AuditLocation(disk_file._datadir, 'sda', '0', @@ -188,7 +188,7 @@ class TestAuditor(unittest.TestCase): def test_object_audit_diff_data(self): auditor_worker = auditor.AuditorWorker(self.conf, self.logger, self.rcache, self.devices) - data = '0' * 1024 + data = b'0' * 1024 etag = md5() timestamp = str(normalize_timestamp(time.time())) with self.disk_file.create() as writer: @@ -212,9 +212,7 @@ class TestAuditor(unittest.TestCase): AuditLocation(self.disk_file._datadir, 'sda', '0', policy=POLICIES.legacy)) self.assertEqual(auditor_worker.quarantines, pre_quarantines) - etag = md5() - etag.update('1' + '0' * 1023) - etag = etag.hexdigest() + etag = md5(b'1' + b'0' * 1023).hexdigest() metadata['ETag'] = etag with self.disk_file.create() as writer: @@ -231,8 +229,8 @@ class TestAuditor(unittest.TestCase): timestamp = str(normalize_timestamp(time.time())) path = os.path.join(self.disk_file._datadir, timestamp + '.data') mkdirs(self.disk_file._datadir) - fp = open(path, 'w') - fp.write('0' * 1024) + fp = open(path, 'wb') + fp.write(b'0' * 1024) fp.close() invalidate_hash(os.path.dirname(self.disk_file._datadir)) auditor_worker = auditor.AuditorWorker(self.conf, self.logger, @@ -362,7 +360,7 @@ class TestAuditor(unittest.TestCase): location = AuditLocation(self.disk_file._datadir, 'sda', '0', policy=self.disk_file.policy) - data = 'VERIFY' + data = b'VERIFY' etag = md5() timestamp = str(normalize_timestamp(time.time())) with self.disk_file.create() as writer: @@ -440,7 +438,7 @@ class TestAuditor(unittest.TestCase): auditor_worker.last_logged = time.time() timestamp = str(normalize_timestamp(time.time())) pre_errors = auditor_worker.errors - data = '0' * 1024 + data = b'0' * 1024 etag = md5() with self.disk_file.create() as writer: writer.write(data) @@ -464,7 +462,7 @@ class TestAuditor(unittest.TestCase): auditor_worker.log_time = 0 timestamp = str(normalize_timestamp(time.time())) pre_quarantines = auditor_worker.quarantines - data = '0' * 1024 + data = b'0' * 1024 def write_file(df): with df.create() as writer: @@ -491,7 +489,7 @@ class TestAuditor(unittest.TestCase): self.assertEqual(auditor_worker.stats_buckets[10240], 0) # pick up some additional code coverage, large file - data = '0' * 1024 * 1024 + data = b'0' * 1024 * 1024 for df in (self.disk_file, self.disk_file_ec): with df.create() as writer: writer.write(data) @@ -545,7 +543,7 @@ class TestAuditor(unittest.TestCase): pre_quarantines = auditor_worker.quarantines # pretend that we logged (and reset counters) just now auditor_worker.last_logged = time.time() - data = '0' * 1024 + data = b'0' * 1024 etag = md5() with self.disk_file.create() as writer: writer.write(data) @@ -557,7 +555,7 @@ class TestAuditor(unittest.TestCase): 'Content-Length': str(os.fstat(writer._fd).st_size), } writer.put(metadata) - os.write(writer._fd, 'extra_data') + os.write(writer._fd, b'extra_data') writer.commit(Timestamp(timestamp)) auditor_worker.audit_all_objects() self.assertEqual(auditor_worker.quarantines, pre_quarantines + 1) @@ -569,7 +567,7 @@ class TestAuditor(unittest.TestCase): auditor_worker.last_logged = time.time() timestamp = str(normalize_timestamp(time.time())) pre_quarantines = auditor_worker.quarantines - data = '0' * 10 + data = b'0' * 10 etag = md5() with self.disk_file.create() as writer: writer.write(data) @@ -585,7 +583,7 @@ class TestAuditor(unittest.TestCase): auditor_worker.audit_all_objects() self.disk_file = self.df_mgr.get_diskfile('sda', '0', 'a', 'c', 'ob', policy=POLICIES.legacy) - data = '1' * 10 + data = b'1' * 10 etag = md5() with self.disk_file.create() as writer: writer.write(data) @@ -598,14 +596,14 @@ class TestAuditor(unittest.TestCase): } writer.put(metadata) writer.commit(Timestamp(timestamp)) - os.write(writer._fd, 'extra_data') + os.write(writer._fd, b'extra_data') auditor_worker.audit_all_objects() self.assertEqual(auditor_worker.quarantines, pre_quarantines + 1) def test_object_run_fast_track_non_zero(self): self.auditor = auditor.ObjectAuditor(self.conf) self.auditor.log_time = 0 - data = '0' * 1024 + data = b'0' * 1024 etag = md5() with self.disk_file.create() as writer: writer.write(data) @@ -620,7 +618,7 @@ class TestAuditor(unittest.TestCase): writer.put(metadata) writer.commit(Timestamp(timestamp)) etag = md5() - etag.update('1' + '0' * 1023) + etag.update(b'1' + b'0' * 1023) etag = etag.hexdigest() metadata['ETag'] = etag write_metadata(writer._fd, metadata)