openstack/swift
Code Issues Proposed changes

Merge "Merge repeat code for rebalance"

Browse Source
This commit is contained in:
Zuul 2018-01-15 15:05:01 +00:00 committed by Gerrit Code Review
commit 909e371515
2 changed files with 117 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
swift/common/ring/builder.py
View File

@ -1369,6 +1369,16 @@ class RingBuilder(object):
def _sort_key_for(dev): def _sort_key_for(dev):
return (dev['parts_wanted'], random.randint(0, 0xFFFF), dev['id']) return (dev['parts_wanted'], random.randint(0, 0xFFFF), dev['id'])
def _validate_replicas_at_tier(self, replicas_by_tier):
tiers = ['cluster', 'regions', 'zones', 'servers', 'devices']
for i, tier_name in enumerate(tiers):
replicas_at_tier = sum(replicas_by_tier[t] for t in
replicas_by_tier if len(t) == i)
if abs(self.replicas - replicas_at_tier) > 1e-10:
raise exceptions.RingValidationError(
'%s != %s at tier %s' % (
replicas_at_tier, self.replicas, tier_name))
def _build_max_replicas_by_tier(self, bound=math.ceil): def _build_max_replicas_by_tier(self, bound=math.ceil):
""" """
Returns a defaultdict of (tier: replica_count) for all tiers in the Returns a defaultdict of (tier: replica_count) for all tiers in the
@ -1483,14 +1493,7 @@ class RingBuilder(object):
# belts & suspenders/paranoia - at every level, the sum of # belts & suspenders/paranoia - at every level, the sum of
# weighted_replicas should be very close to the total number of # weighted_replicas should be very close to the total number of
# replicas for the ring # replicas for the ring
tiers = ['cluster', 'regions', 'zones', 'servers', 'devices'] self._validate_replicas_at_tier(weighted_replicas_by_tier)
for i, tier_name in enumerate(tiers):
replicas_at_tier = sum(weighted_replicas_by_tier[t] for t in
weighted_replicas_by_tier if len(t) == i)
if abs(self.replicas - replicas_at_tier) > 1e-10:
raise exceptions.RingValidationError(
'%s != %s at tier %s' % (
replicas_at_tier, self.replicas, tier_name))
return weighted_replicas_by_tier return weighted_replicas_by_tier
@ -1593,14 +1596,7 @@ class RingBuilder(object):
# belts & suspenders/paranoia - at every level, the sum of # belts & suspenders/paranoia - at every level, the sum of
# wanted_replicas should be very close to the total number of # wanted_replicas should be very close to the total number of
# replicas for the ring # replicas for the ring
tiers = ['cluster', 'regions', 'zones', 'servers', 'devices'] self._validate_replicas_at_tier(wanted_replicas)
for i, tier_name in enumerate(tiers):
replicas_at_tier = sum(wanted_replicas[t] for t in
wanted_replicas if len(t) == i)
if abs(self.replicas - replicas_at_tier) > 1e-10:
raise exceptions.RingValidationError(
'%s != %s at tier %s' % (
replicas_at_tier, self.replicas, tier_name))
return wanted_replicas return wanted_replicas
@ -1629,14 +1625,7 @@ class RingBuilder(object):
# belts & suspenders/paranoia - at every level, the sum of # belts & suspenders/paranoia - at every level, the sum of
# target_replicas should be very close to the total number # target_replicas should be very close to the total number
# of replicas for the ring # of replicas for the ring
tiers = ['cluster', 'regions', 'zones', 'servers', 'devices'] self._validate_replicas_at_tier(target_replicas)
for i, tier_name in enumerate(tiers):
replicas_at_tier = sum(target_replicas[t] for t in
target_replicas if len(t) == i)
if abs(self.replicas - replicas_at_tier) > 1e-10:
raise exceptions.RingValidationError(
'%s != %s at tier %s' % (
replicas_at_tier, self.replicas, tier_name))
return target_replicas return target_replicas

104
test/unit/common/ring/test_builder.py
View File

@ -446,6 +446,110 @@ class TestRingBuilder(unittest.TestCase):
# maybe not *perfect*, but should be close # maybe not *perfect*, but should be close
self.assertLessEqual(balance, 1) self.assertLessEqual(balance, 1)
def test_validate_replicate_by_tier(self):
rb = ring.RingBuilder(5, 3, 0)
# replicas 3.0 and three devices with two weight 10 and one weight 5
rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 10,
'ip': '127.0.0.1', 'port': 6000, 'device': 'object1'})
rb.add_dev({'id': 1, 'region': 1, 'zone': 1, 'weight': 10,
'ip': '127.0.0.1', 'port': 6000, 'device': 'object2'})
rb.add_dev({'id': 2, 'region': 2, 'zone': 2, 'weight': 5,
'ip': '127.0.0.1', 'port': 6000, 'device': 'object3'})
rb.rebalance()
# validate weighted_replicas
weighted_replicas = \
defaultdict(float,
{(0, 0): 1.0, (1,): 1.0,
(2,): 0.9999999999999999,
(0, 0, '127.0.0.1', 0): 1.0,
(1, 1, '127.0.0.1', 1): 1.0,
(0, 0, '127.0.0.1'): 1.0,
(1, 1, '127.0.0.1'): 1.0, (): 3.0,
(2, 2, '127.0.0.1'): 0.9999999999999999,
(2, 2): 0.9999999999999999,
(2, 2, '127.0.0.1', 2): 0.9999999999999999,
(1, 1): 1.0, (0,): 1.0
})
try:
rb._validate_replicas_at_tier(weighted_replicas)
except Exception as e:
self.fail('weighted_replicas is invalid for %s' % e)
# validate wanted_replicas
wanted_replicas = \
defaultdict(float,
{(0, 0): 1.0, (1,): 1.0, (2,): 1.0,
(0, 0, '127.0.0.1', 0): 1.0,
(1, 1, '127.0.0.1', 1): 1.0,
(0, 0, '127.0.0.1'): 1.0,
(1, 1, '127.0.0.1'): 1.0, (): 3.0,
(2, 2, '127.0.0.1'): 1.0,
(2, 2): 1.0, (2, 2, '127.0.0.1', 2): 1.0,
(1, 1): 1.0, (0,): 1.0
})
try:
rb._validate_replicas_at_tier(wanted_replicas)
except Exception as e:
self.fail('wanted_replicas is invalid for %s' % e)
# validate target_replicas
target_replicas = \
defaultdict(float,
{(0, 0): 1.0, (1,): 1.0,
(2,): 0.9999999999999999,
(0, 0, '127.0.0.1', 0): 1.0,
(1, 1, '127.0.0.1', 1): 1.0,
(0, 0, '127.0.0.1'): 1.0,
(1, 1, '127.0.0.1'): 1.0,
(2, 2, '127.0.0.1'): 0.9999999999999999,
(2, 2): 0.9999999999999999, (): 3.0,
(2, 2, '127.0.0.1', 2): 0.9999999999999999,
(1, 1): 1.0, (0,): 1.0
})
try:
rb._validate_replicas_at_tier(target_replicas)
except Exception as e:
self.fail('target_replicas is invalid for %s' % e)
# set overload to 10%
rb.set_overload(0.1)
rb.rebalance()
# validate target_replicas
target_replicas = \
defaultdict(float,
{(0, 0): 1.0, (1,): 1.0, (2,): 1.0,
(0, 0, '127.0.0.1', 0): 1.0,
(1, 1, '127.0.0.1', 1): 1.0,
(0, 0, '127.0.0.1'): 1.0,
(1, 1, '127.0.0.1'): 1.0,
(2, 2, '127.0.0.1'): 1.0, (2, 2): 1.0, (): 3.0,
(2, 2, '127.0.0.1', 2): 1.0,
(1, 1): 1.0, (0,): 1.0
})
try:
rb._validate_replicas_at_tier(target_replicas)
except Exception as e:
self.fail('target_replicas is invalid after overload for %s'
% e)
# invalidate case
pseudo_replicas = \
defaultdict(float,
{(0, 0): 1.1, (1,): 1.0, (2,): 1.0,
(0, 0, '127.0.0.1', 0): 1.0,
(1, 1, '127.0.0.1', 1): 1.0,
(0, 0, '127.0.0.1'): 1.0,
(1, 1, '127.0.0.1'): 1.0,
(2, 2, '127.0.0.1'): 1.0,
(2, 2): 1.0, (): 3.0,
(2, 2, '127.0.0.1', 2): 1.0,
(1, 1): 1.0, (0,): 1.0
})
with self.assertRaises(exceptions.RingValidationError) as ctx:
rb._validate_replicas_at_tier(pseudo_replicas)
self.assertEqual('3.1 != 3 at tier zones', str(ctx.exception))
def test_multitier_partial(self): def test_multitier_partial(self):
# Multitier test, nothing full # Multitier test, nothing full
rb = ring.RingBuilder(8, 3, 1) rb = ring.RingBuilder(8, 3, 1)