diff --git a/swift/common/ring/ring.py b/swift/common/ring/ring.py index 9f3d4f7b1b..884aa92399 100644 --- a/swift/common/ring/ring.py +++ b/swift/common/ring/ring.py @@ -26,6 +26,7 @@ from io import BufferedReader from hashlib import md5 from itertools import chain from tempfile import NamedTemporaryFile +import sys from six.moves import range @@ -70,10 +71,15 @@ class RingData(object): if metadata_only: return ring_dict + byteswap = (ring_dict.get('byteorder', sys.byteorder) != sys.byteorder) + partition_count = 1 << (32 - ring_dict['part_shift']) for x in range(ring_dict['replica_count']): - ring_dict['replica2part2dev_id'].append( - array.array('H', gz_file.read(2 * partition_count))) + part2dev = array.array('H', gz_file.read(2 * partition_count)) + if byteswap: + part2dev.byteswap() + ring_dict['replica2part2dev_id'].append(part2dev) + return ring_dict @classmethod @@ -117,7 +123,8 @@ class RingData(object): json_encoder = json.JSONEncoder(sort_keys=True) json_text = json_encoder.encode( {'devs': ring['devs'], 'part_shift': ring['part_shift'], - 'replica_count': len(ring['replica2part2dev_id'])}) + 'replica_count': len(ring['replica2part2dev_id']), + 'byteorder': sys.byteorder}) json_len = len(json_text) file_obj.write(struct.pack('!I', json_len)) file_obj.write(json_text) diff --git a/test/unit/common/middleware/test_recon.py b/test/unit/common/middleware/test_recon.py index 63e82cd61e..aa4d6499e9 100644 --- a/test/unit/common/middleware/test_recon.py +++ b/test/unit/common/middleware/test_recon.py @@ -21,6 +21,7 @@ from posix import stat_result, statvfs_result from shutil import rmtree import unittest from unittest import TestCase +import sys from swift import __version__ as swiftver from swift.common import ring, utils @@ -313,23 +314,33 @@ class TestReconSuccess(TestCase): def test_get_ring_md5(self): # We should only see configured and present rings, so to handle the # "normal" case just patch the policies to match the existing rings. - expt_out = {'%s/account.ring.gz' % self.tempdir: - '11e0c98abb209474d40d6a9a8a523803', - '%s/container.ring.gz' % self.tempdir: - '6685496a4045ce0be123068e0165a64d', - '%s/object.ring.gz' % self.tempdir: - '782728be98644fb725e165d4bf5728d4', - '%s/object-1.ring.gz' % self.tempdir: - '7c3a4bc9f724d4eb69c9b797cdc28b8c', - '%s/object-2.ring.gz' % self.tempdir: - '324b9c4da20cf7ef097edbd219d296e0'} + expt_out = {'little': {'%s/account.ring.gz' % self.tempdir: + '672c6c50dfcb77e04f5a2124aef87596', + '%s/container.ring.gz' % self.tempdir: + '4c4392f8bf816596990ca7cd4d4b6e50', + '%s/object.ring.gz' % self.tempdir: + 'a34178f7399706e41395eb3ac8b2c4f3', + '%s/object-1.ring.gz' % self.tempdir: + 'cd54c473676ce5e3103e68f0e9f2326d', + '%s/object-2.ring.gz' % self.tempdir: + '8783ec76f29bbcfd3f51acc63b2fc337'}, + 'big': {'%s/account.ring.gz' % self.tempdir: + '70d2dde8144c09e5b42858e0fa17ab7e', + '%s/container.ring.gz' % self.tempdir: + '0bd14f4327cbea88bde7ab7850d4d77d', + '%s/object.ring.gz' % self.tempdir: + '2d17555687af36fd8c7ee5b0c492c582', + '%s/object-1.ring.gz' % self.tempdir: + 'bc1145d31771d7957d939077fe40e2e8', + '%s/object-2.ring.gz' % self.tempdir: + 'cd95a01ae1ab158f2e9e4c207aeb1769'}} # We need to instantiate app after overriding the configured policies. # object-{1,2}.ring.gz should both appear as they are present on disk # and were configured as policies. app = recon.ReconMiddleware(FakeApp(), {'swift_dir': self.tempdir}) self.assertEqual(sorted(app.get_ring_md5().items()), - sorted(expt_out.items())) + sorted(expt_out[sys.byteorder].items())) def test_get_ring_md5_ioerror_produces_none_hash(self): # Ring files that are present but produce an IOError on read should @@ -369,13 +380,18 @@ class TestReconSuccess(TestCase): raise IOError return open(fn, fmode) - expt_out = {'%s/account.ring.gz' % self.tempdir: None, - '%s/container.ring.gz' % self.tempdir: None, - '%s/object.ring.gz' % self.tempdir: - '782728be98644fb725e165d4bf5728d4'} + expt_out = {'little': {'%s/account.ring.gz' % self.tempdir: None, + '%s/container.ring.gz' % self.tempdir: None, + '%s/object.ring.gz' % self.tempdir: + 'a34178f7399706e41395eb3ac8b2c4f3'}, + 'big': {'%s/account.ring.gz' % self.tempdir: None, + '%s/container.ring.gz' % self.tempdir: None, + '%s/object.ring.gz' % self.tempdir: + '2d17555687af36fd8c7ee5b0c492c582'}} + ringmd5 = self.app.get_ring_md5(openr=fake_open_objonly) self.assertEqual(sorted(ringmd5.items()), - sorted(expt_out.items())) + sorted(expt_out[sys.byteorder].items())) @patch_policies([ StoragePolicy(0, 'stagecoach'), @@ -386,14 +402,22 @@ class TestReconSuccess(TestCase): # If a configured ring is missing when the app is instantiated, but is # later moved into place, we shouldn't need to restart object-server # for it to appear in recon. - expt_out = {'%s/account.ring.gz' % self.tempdir: - '11e0c98abb209474d40d6a9a8a523803', - '%s/container.ring.gz' % self.tempdir: - '6685496a4045ce0be123068e0165a64d', - '%s/object.ring.gz' % self.tempdir: - '782728be98644fb725e165d4bf5728d4', - '%s/object-2.ring.gz' % self.tempdir: - '324b9c4da20cf7ef097edbd219d296e0'} + expt_out = {'little': {'%s/account.ring.gz' % self.tempdir: + '672c6c50dfcb77e04f5a2124aef87596', + '%s/container.ring.gz' % self.tempdir: + '4c4392f8bf816596990ca7cd4d4b6e50', + '%s/object.ring.gz' % self.tempdir: + 'a34178f7399706e41395eb3ac8b2c4f3', + '%s/object-2.ring.gz' % self.tempdir: + '8783ec76f29bbcfd3f51acc63b2fc337'}, + 'big': {'%s/account.ring.gz' % self.tempdir: + '70d2dde8144c09e5b42858e0fa17ab7e', + '%s/container.ring.gz' % self.tempdir: + '0bd14f4327cbea88bde7ab7850d4d77d', + '%s/object.ring.gz' % self.tempdir: + '2d17555687af36fd8c7ee5b0c492c582', + '%s/object-2.ring.gz' % self.tempdir: + 'cd95a01ae1ab158f2e9e4c207aeb1769'}} # We need to instantiate app after overriding the configured policies. # object-1.ring.gz should not appear as it's present but unconfigured. @@ -401,7 +425,7 @@ class TestReconSuccess(TestCase): # present. app = recon.ReconMiddleware(FakeApp(), {'swift_dir': self.tempdir}) self.assertEqual(sorted(app.get_ring_md5().items()), - sorted(expt_out.items())) + sorted(expt_out[sys.byteorder].items())) # Simulate the configured policy's missing ringfile being moved into # place during runtime @@ -412,12 +436,14 @@ class TestReconSuccess(TestCase): array.array('H', [1, 1, 0, 3])] self._create_ring(os.path.join(self.tempdir, ringfn), ringmap, self.ring_devs, self.ring_part_shift) - expt_out[ringpath] = 'a7e591642beea6933f64aebd56f357d9' + expt_out[sys.byteorder][ringpath] = \ + '77f752964f3bd4719e1b9b6cee47659f' if sys.byteorder == 'little' \ + else '3e189e6c668a4d159707438dfb87589a' # We should now see it in the ringmd5 response, without a restart # (using the same app instance) self.assertEqual(sorted(app.get_ring_md5().items()), - sorted(expt_out.items())) + sorted(expt_out[sys.byteorder].items())) @patch_policies([ StoragePolicy(0, 'stagecoach', is_default=True), @@ -427,14 +453,22 @@ class TestReconSuccess(TestCase): def test_get_ring_md5_excludes_configured_missing_obj_rings(self): # Object rings that are configured but missing aren't meant to appear # in the ringmd5 response. - expt_out = {'%s/account.ring.gz' % self.tempdir: - '11e0c98abb209474d40d6a9a8a523803', - '%s/container.ring.gz' % self.tempdir: - '6685496a4045ce0be123068e0165a64d', - '%s/object.ring.gz' % self.tempdir: - '782728be98644fb725e165d4bf5728d4', - '%s/object-2.ring.gz' % self.tempdir: - '324b9c4da20cf7ef097edbd219d296e0'} + expt_out = {'little': {'%s/account.ring.gz' % self.tempdir: + '672c6c50dfcb77e04f5a2124aef87596', + '%s/container.ring.gz' % self.tempdir: + '4c4392f8bf816596990ca7cd4d4b6e50', + '%s/object.ring.gz' % self.tempdir: + 'a34178f7399706e41395eb3ac8b2c4f3', + '%s/object-2.ring.gz' % self.tempdir: + '8783ec76f29bbcfd3f51acc63b2fc337'}, + 'big': {'%s/account.ring.gz' % self.tempdir: + '70d2dde8144c09e5b42858e0fa17ab7e', + '%s/container.ring.gz' % self.tempdir: + '0bd14f4327cbea88bde7ab7850d4d77d', + '%s/object.ring.gz' % self.tempdir: + '2d17555687af36fd8c7ee5b0c492c582', + '%s/object-2.ring.gz' % self.tempdir: + 'cd95a01ae1ab158f2e9e4c207aeb1769'}} # We need to instantiate app after overriding the configured policies. # object-1.ring.gz should not appear as it's present but unconfigured. @@ -442,7 +476,7 @@ class TestReconSuccess(TestCase): # present. app = recon.ReconMiddleware(FakeApp(), {'swift_dir': self.tempdir}) self.assertEqual(sorted(app.get_ring_md5().items()), - sorted(expt_out.items())) + sorted(expt_out[sys.byteorder].items())) @patch_policies([ StoragePolicy(0, 'zero', is_default=True), @@ -450,19 +484,25 @@ class TestReconSuccess(TestCase): def test_get_ring_md5_excludes_unconfigured_present_obj_rings(self): # Object rings that are present but not configured in swift.conf # aren't meant to appear in the ringmd5 response. - expt_out = {'%s/account.ring.gz' % self.tempdir: - '11e0c98abb209474d40d6a9a8a523803', - '%s/container.ring.gz' % self.tempdir: - '6685496a4045ce0be123068e0165a64d', - '%s/object.ring.gz' % self.tempdir: - '782728be98644fb725e165d4bf5728d4'} + expt_out = {'little': {'%s/account.ring.gz' % self.tempdir: + '672c6c50dfcb77e04f5a2124aef87596', + '%s/container.ring.gz' % self.tempdir: + '4c4392f8bf816596990ca7cd4d4b6e50', + '%s/object.ring.gz' % self.tempdir: + 'a34178f7399706e41395eb3ac8b2c4f3'}, + 'big': {'%s/account.ring.gz' % self.tempdir: + '70d2dde8144c09e5b42858e0fa17ab7e', + '%s/container.ring.gz' % self.tempdir: + '0bd14f4327cbea88bde7ab7850d4d77d', + '%s/object.ring.gz' % self.tempdir: + '2d17555687af36fd8c7ee5b0c492c582'}} # We need to instantiate app after overriding the configured policies. # object-{1,2}.ring.gz should not appear as they are present on disk # but were not configured as policies. app = recon.ReconMiddleware(FakeApp(), {'swift_dir': self.tempdir}) self.assertEqual(sorted(app.get_ring_md5().items()), - sorted(expt_out.items())) + sorted(expt_out[sys.byteorder].items())) def test_from_recon_cache(self): oart = OpenAndReadTester(['{"notneeded": 5, "testkey1": "canhazio"}']) diff --git a/test/unit/common/ring/test_ring.py b/test/unit/common/ring/test_ring.py index 7df5b57fab..91b1e5a138 100644 --- a/test/unit/common/ring/test_ring.py +++ b/test/unit/common/ring/test_ring.py @@ -24,6 +24,9 @@ from tempfile import mkdtemp from shutil import rmtree from time import sleep, time import random +import sys +import copy +import mock from six.moves import range @@ -107,6 +110,30 @@ class TestRingData(unittest.TestCase): rd2 = ring.RingData.load(ring_fname) self.assert_ring_data_equal(rd, rd2) + def test_byteswapped_serialization(self): + # Manually byte swap a ring and write it out, claiming it was written + # on a different endian machine. Then read it back in and see if it's + # the same as the non-byte swapped original. + + ring_fname = os.path.join(self.testdir, 'foo.ring.gz') + data = [array.array('H', [0, 1, 0, 1]), array.array('H', [0, 1, 0, 1])] + swapped_data = copy.deepcopy(data) + for x in swapped_data: + x.byteswap() + + with mock.patch.object(sys, 'byteorder', + 'big' if sys.byteorder == 'little' + else 'little'): + rds = ring.RingData(swapped_data, + [{'id': 0, 'zone': 0}, {'id': 1, 'zone': 1}], + 30) + rds.save(ring_fname) + + rd1 = ring.RingData(data, [{'id': 0, 'zone': 0}, {'id': 1, 'zone': 1}], + 30) + rd2 = ring.RingData.load(ring_fname) + self.assert_ring_data_equal(rd1, rd2) + def test_deterministic_serialization(self): """ Two identical rings should produce identical .gz files on disk.