openstack/swift
Code Issues Proposed changes

Merge "sharder: support rows_per_shard in config file"

Browse Source
This commit is contained in:
Zuul 2021-07-07 23:06:08 +00:00 committed by Gerrit Code Review
commit 8066efb43a
6 changed files with 45 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
doc/source/config/container_server_config.rst
View File

@ -319,13 +319,15 @@ shard_container_threshold 1000000 This defines the
enabled will start to
shard. This also
indirectly determines the
initial nominal size of
shard containers, which
is shard_container_threshold//2,
as well as determining
the thresholds for
shrinking and merging
shard containers.
the defaults for
rows_per_shard,
shrink_threshold and
expansion_limit.
rows_per_shard 500000 This defines the initial
nominal size of shard
containers. The default
is shard_container_threshold // 2.
shrink_threshold This defines the
object count below which

10
etc/container-server.conf-sample
View File

@ -366,12 +366,14 @@ use = egg:swift#xprofile
#
# When auto-sharding is enabled shard_container_threshold defines the object
# count at which a container with container-sharding enabled will start to
# shard. shard_container_threshold also indirectly determines the initial
# nominal size of shard containers, which is shard_container_threshold // 2, as
# well as determining the thresholds for shrinking and merging shard
# containers.
# shard. shard_container_threshold also indirectly determines the defaults for
# rows_per_shard, shrink_threshold and expansion_limit.
# shard_container_threshold = 1000000
#
# rows_per_shard determines the initial nominal size of shard containers. The
# default is shard_container_threshold // 2
# rows_per_shard = 500000
#
# When auto-sharding is enabled shrink_threshold defines the object count
# below which a 'donor' shard container will be considered for shrinking into
# another 'acceptor' shard container. The default is determined by

9
swift/cli/manage_shard_ranges.py
View File

@ -754,7 +754,14 @@ def _make_parser():
'subcommand, a shard data file.')
parser.add_argument('--config', dest='conf_file', required=False,
help='Path to config file with [container-sharder] '
'section')
'section. The following subcommand options will '
'be loaded from a config file if they are not '
'given on the command line: '
'rows_per_shard, '
'max_shrinking, '
'max_expanding, '
'shrink_threshold, '
'expansion_limit')
parser.add_argument('--verbose', '-v', action='count', default=0,
help='Increase output verbosity')
# this is useful for probe tests that shard containers with unrealistically

4
swift/container/sharder.py
View File

@ -627,7 +627,9 @@ class ContainerSharderConf(object):
'shrink_threshold', int, self.shrink_threshold)
self.expansion_limit = get_val(
'expansion_limit', int, self.expansion_limit)
self.rows_per_shard = self.shard_container_threshold // 2
self.rows_per_shard = get_val(
'rows_per_shard', config_positive_int_value,
max(self.shard_container_threshold // 2, 1))
def percent_of_threshold(self, val):
return int(config_percent_value(val) * self.shard_container_threshold)

3
test/unit/cli/test_manage_shard_ranges.py
View File

@ -139,6 +139,7 @@ class TestManageShardRanges(unittest.TestCase):
shrink_threshold = 150
expansion_limit = 650
shard_container_threshold = 1000
rows_per_shard = 600
max_shrinking = 33
max_expanding = 31
"""
@ -169,7 +170,7 @@ class TestManageShardRanges(unittest.TestCase):
expected = Namespace(conf_file=conf_file,
path_to_file=mock.ANY,
func=mock.ANY,
rows_per_shard=500,
rows_per_shard=600,
subcommand='find',
force_commits=False,
verbose=0)

21
test/unit/container/test_sharder.py
View File

@ -186,6 +186,20 @@ class TestSharder(BaseTestSharder):
allow_modify_pipeline=False,
use_replication_network=True)
# non-default shard_container_threshold influences other defaults
conf = {'shard_container_threshold': 20000000}
expected.update({
'shard_container_threshold': 20000000,
'shrink_threshold': 2000000,
'expansion_limit': 15000000,
'rows_per_shard': 10000000
})
sharder, mock_ic = self._do_test_init(conf, expected)
mock_ic.assert_called_once_with(
'/etc/swift/internal-client.conf', 'Swift Container Sharder', 3,
allow_modify_pipeline=False,
use_replication_network=True)
# non-default values
conf = {
'mount_check': False, 'bind_ip': '10.11.12.13', 'bind_port': 62010,
@ -212,7 +226,7 @@ class TestSharder(BaseTestSharder):
'existing_shard_replication_quorum': 0,
'max_shrinking': 5,
'max_expanding': 4,
'rows_per_shard': 13, # should be ignored - not configurable
'rows_per_shard': 13
}
expected = {
'mount_check': False, 'bind_ip': '10.11.12.13', 'port': 62010,
@ -224,7 +238,7 @@ class TestSharder(BaseTestSharder):
'rsync_module': '{replication_ip}::container_sda',
'reclaim_age': 86400 * 14,
'shard_container_threshold': 20000000,
'rows_per_shard': 10000000,
'rows_per_shard': 13,
'shrink_threshold': 7000000,
'expansion_limit': 17000000,
'cleave_batch_size': 4,
@ -7293,8 +7307,7 @@ class TestContainerSharderConf(unittest.TestCase):
'shrink_threshold': 100001,
'expansion_limit': 750001,
'rows_per_shard': 500001}
# rows_per_shard is not directly configurable
expected = dict(conf, rows_per_shard=1000000)
expected = dict(conf)
conf.update({'unexpected': 'option'})
self.assertEqual(expected, vars(ContainerSharderConf(conf)))