diff --git a/doc/source/config/container_server_config.rst b/doc/source/config/container_server_config.rst index 8b64214fb2..1684acfcaf 100644 --- a/doc/source/config/container_server_config.rst +++ b/doc/source/config/container_server_config.rst @@ -319,13 +319,15 @@ shard_container_threshold 1000000 This defines the enabled will start to shard. This also indirectly determines the - initial nominal size of - shard containers, which - is shard_container_threshold//2, - as well as determining - the thresholds for - shrinking and merging - shard containers. + the defaults for + rows_per_shard, + shrink_threshold and + expansion_limit. + +rows_per_shard 500000 This defines the initial + nominal size of shard + containers. The default + is shard_container_threshold // 2. shrink_threshold This defines the object count below which diff --git a/etc/container-server.conf-sample b/etc/container-server.conf-sample index d8a6a81a54..262010e429 100644 --- a/etc/container-server.conf-sample +++ b/etc/container-server.conf-sample @@ -366,12 +366,14 @@ use = egg:swift#xprofile # # When auto-sharding is enabled shard_container_threshold defines the object # count at which a container with container-sharding enabled will start to -# shard. shard_container_threshold also indirectly determines the initial -# nominal size of shard containers, which is shard_container_threshold // 2, as -# well as determining the thresholds for shrinking and merging shard -# containers. +# shard. shard_container_threshold also indirectly determines the defaults for +# rows_per_shard, shrink_threshold and expansion_limit. # shard_container_threshold = 1000000 # +# rows_per_shard determines the initial nominal size of shard containers. The +# default is shard_container_threshold // 2 +# rows_per_shard = 500000 +# # When auto-sharding is enabled shrink_threshold defines the object count # below which a 'donor' shard container will be considered for shrinking into # another 'acceptor' shard container. The default is determined by diff --git a/swift/cli/manage_shard_ranges.py b/swift/cli/manage_shard_ranges.py index 3ec4f16d64..9ecdcf1a6d 100644 --- a/swift/cli/manage_shard_ranges.py +++ b/swift/cli/manage_shard_ranges.py @@ -754,7 +754,14 @@ def _make_parser(): 'subcommand, a shard data file.') parser.add_argument('--config', dest='conf_file', required=False, help='Path to config file with [container-sharder] ' - 'section') + 'section. The following subcommand options will ' + 'be loaded from a config file if they are not ' + 'given on the command line: ' + 'rows_per_shard, ' + 'max_shrinking, ' + 'max_expanding, ' + 'shrink_threshold, ' + 'expansion_limit') parser.add_argument('--verbose', '-v', action='count', default=0, help='Increase output verbosity') # this is useful for probe tests that shard containers with unrealistically diff --git a/swift/container/sharder.py b/swift/container/sharder.py index 4a243deabd..8b44524cba 100644 --- a/swift/container/sharder.py +++ b/swift/container/sharder.py @@ -627,7 +627,9 @@ class ContainerSharderConf(object): 'shrink_threshold', int, self.shrink_threshold) self.expansion_limit = get_val( 'expansion_limit', int, self.expansion_limit) - self.rows_per_shard = self.shard_container_threshold // 2 + self.rows_per_shard = get_val( + 'rows_per_shard', config_positive_int_value, + max(self.shard_container_threshold // 2, 1)) def percent_of_threshold(self, val): return int(config_percent_value(val) * self.shard_container_threshold) diff --git a/test/unit/cli/test_manage_shard_ranges.py b/test/unit/cli/test_manage_shard_ranges.py index 91ee18505b..79e84251f0 100644 --- a/test/unit/cli/test_manage_shard_ranges.py +++ b/test/unit/cli/test_manage_shard_ranges.py @@ -139,6 +139,7 @@ class TestManageShardRanges(unittest.TestCase): shrink_threshold = 150 expansion_limit = 650 shard_container_threshold = 1000 + rows_per_shard = 600 max_shrinking = 33 max_expanding = 31 """ @@ -169,7 +170,7 @@ class TestManageShardRanges(unittest.TestCase): expected = Namespace(conf_file=conf_file, path_to_file=mock.ANY, func=mock.ANY, - rows_per_shard=500, + rows_per_shard=600, subcommand='find', force_commits=False, verbose=0) diff --git a/test/unit/container/test_sharder.py b/test/unit/container/test_sharder.py index 17120c2b57..8dd632250c 100644 --- a/test/unit/container/test_sharder.py +++ b/test/unit/container/test_sharder.py @@ -186,6 +186,20 @@ class TestSharder(BaseTestSharder): allow_modify_pipeline=False, use_replication_network=True) + # non-default shard_container_threshold influences other defaults + conf = {'shard_container_threshold': 20000000} + expected.update({ + 'shard_container_threshold': 20000000, + 'shrink_threshold': 2000000, + 'expansion_limit': 15000000, + 'rows_per_shard': 10000000 + }) + sharder, mock_ic = self._do_test_init(conf, expected) + mock_ic.assert_called_once_with( + '/etc/swift/internal-client.conf', 'Swift Container Sharder', 3, + allow_modify_pipeline=False, + use_replication_network=True) + # non-default values conf = { 'mount_check': False, 'bind_ip': '10.11.12.13', 'bind_port': 62010, @@ -212,7 +226,7 @@ class TestSharder(BaseTestSharder): 'existing_shard_replication_quorum': 0, 'max_shrinking': 5, 'max_expanding': 4, - 'rows_per_shard': 13, # should be ignored - not configurable + 'rows_per_shard': 13 } expected = { 'mount_check': False, 'bind_ip': '10.11.12.13', 'port': 62010, @@ -224,7 +238,7 @@ class TestSharder(BaseTestSharder): 'rsync_module': '{replication_ip}::container_sda', 'reclaim_age': 86400 * 14, 'shard_container_threshold': 20000000, - 'rows_per_shard': 10000000, + 'rows_per_shard': 13, 'shrink_threshold': 7000000, 'expansion_limit': 17000000, 'cleave_batch_size': 4, @@ -7293,8 +7307,7 @@ class TestContainerSharderConf(unittest.TestCase): 'shrink_threshold': 100001, 'expansion_limit': 750001, 'rows_per_shard': 500001} - # rows_per_shard is not directly configurable - expected = dict(conf, rows_per_shard=1000000) + expected = dict(conf) conf.update({'unexpected': 'option'}) self.assertEqual(expected, vars(ContainerSharderConf(conf)))