openstack/swift
Code Issues Proposed changes

Merge "sharder: report perfectly overlapping shard ranges"

Browse Source
This commit is contained in:
Zuul 2021-05-17 08:15:14 +00:00 committed by Gerrit Code Review
commit 0a58574109
2 changed files with 20 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
swift/container/sharder.py
View File

@ -108,12 +108,13 @@ def find_overlapping_ranges(shard_ranges):
each other. each other.
""" """
result = set() result = set()
for shard_range in shard_ranges: for i, shard_range in enumerate(shard_ranges):
overlapping = [sr for sr in shard_ranges overlapping = [
if shard_range != sr and shard_range.overlaps(sr)] sr for sr in shard_ranges[i + 1:]
if shard_range.name != sr.name and shard_range.overlaps(sr)]
if overlapping: if overlapping:
overlapping.append(shard_range) overlapping.append(shard_range)
overlapping.sort() overlapping.sort(key=ShardRange.sort_key)
result.add(tuple(overlapping)) result.add(tuple(overlapping))
return result return result
@ -994,12 +995,14 @@ class ContainerSharder(ContainerReplicator):
continue continue
shard_ranges = broker.get_shard_ranges(states=state) shard_ranges = broker.get_shard_ranges(states=state)
overlaps = find_overlapping_ranges(shard_ranges) overlaps = find_overlapping_ranges(shard_ranges)
for overlapping_ranges in overlaps: if overlaps:
all_overlaps = ', '.join(
[' '.join(['%s-%s' % (sr.lower, sr.upper)
for sr in overlapping_ranges])
for overlapping_ranges in sorted(list(overlaps))])
warnings.append( warnings.append(
'overlapping ranges in state %s: %s' % 'overlapping ranges in state %r: %s' %
(ShardRange.STATES[state], (ShardRange.STATES[state], all_overlaps))
' '.join(['%s-%s' % (sr.lower, sr.upper)
for sr in overlapping_ranges])))
if warnings: if warnings:
self.logger.warning( self.logger.warning(

13
test/unit/container/test_sharder.py
View File

@ -117,10 +117,10 @@ class BaseTestSharder(unittest.TestCase):
if not isinstance(state, (tuple, list)): if not isinstance(state, (tuple, list)):
state = [state] * len(bounds) state = [state] * len(bounds)
state_iter = iter(state) state_iter = iter(state)
return [ShardRange('.shards_a/c_%s' % upper, timestamp, return [ShardRange('.shards_a/c_%s_%s' % (upper, index), timestamp,
lower, upper, state=next(state_iter), lower, upper, state=next(state_iter),
object_count=object_count, **kwargs) object_count=object_count, **kwargs)
for lower, upper in bounds] for index, (lower, upper) in enumerate(bounds)]
def ts_encoded(self): def ts_encoded(self):
# make a unique timestamp string with multiple timestamps encoded; # make a unique timestamp string with multiple timestamps encoded;
@ -4859,11 +4859,14 @@ class TestSharder(BaseTestSharder):
self.assertIn( self.assertIn(
'Audit failed for root %s' % broker.db_file, line) 'Audit failed for root %s' % broker.db_file, line)
self.assertIn( self.assertIn(
'overlapping ranges in state %s: k-t s-z' % state_text, 'overlapping ranges in state %r: k-t s-y, y-z y-z'
line) % state_text, line)
# check for no duplicates in reversed order
self.assertNotIn('s-z k-t', line)
expected_stats = {'attempted': 1, 'success': 0, 'failure': 1} expected_stats = {'attempted': 1, 'success': 0, 'failure': 1}
shard_bounds = (('a', 'j'), ('k', 't'), ('s', 'z')) shard_bounds = (('a', 'j'), ('k', 't'), ('s', 'y'),
('y', 'z'), ('y', 'z'))
for state, state_text in ShardRange.STATES.items(): for state, state_text in ShardRange.STATES.items():
if state in (ShardRange.SHRINKING, if state in (ShardRange.SHRINKING,
ShardRange.SHARDED, ShardRange.SHARDED,