Merge "sharder: support rows_per_shard in config file"

doc/source/config/container_server_config.rst

@@ -319,13 +319,15 @@ shard_container_threshold         1000000             This defines the
                                                       enabled will start to
                                                       shard. This also
                                                       indirectly determines the
-                                                      initial nominal size of
+                                                      the defaults for
-                                                      shard containers, which
+                                                      rows_per_shard,
-                                                      is shard_container_threshold//2,
+                                                      shrink_threshold and
-                                                      as well as determining
+                                                      expansion_limit.
-                                                      the thresholds for
+
-                                                      shrinking and merging
+rows_per_shard                    500000              This defines the initial
-                                                      shard containers.
+                                                      nominal size of shard
+                                                      containers. The default
+                                                      is shard_container_threshold // 2.
 
 shrink_threshold                                      This defines the
                                                       object count below which

etc/container-server.conf-sample

@@ -366,12 +366,14 @@ use = egg:swift#xprofile
 #
 # When auto-sharding is enabled shard_container_threshold defines the object
 # count at which a container with container-sharding enabled will start to
-# shard. shard_container_threshold also indirectly determines the initial
+# shard. shard_container_threshold also indirectly determines the defaults for
-# nominal size of shard containers, which is shard_container_threshold // 2, as
+# rows_per_shard, shrink_threshold and expansion_limit.
-# well as determining the thresholds for shrinking and merging shard
-# containers.
 # shard_container_threshold = 1000000
 #
+# rows_per_shard determines the initial nominal size of shard containers. The
+# default is shard_container_threshold // 2
+# rows_per_shard = 500000
+#
 # When auto-sharding is enabled shrink_threshold defines the object count
 # below which a 'donor' shard container will be considered for shrinking into
 # another 'acceptor' shard container. The default is determined by

swift/cli/manage_shard_ranges.py

@@ -754,7 +754,14 @@ def _make_parser():
                         'subcommand, a shard data file.')
     parser.add_argument('--config', dest='conf_file', required=False,
                         help='Path to config file with [container-sharder] '
-                             'section')
+                             'section. The following subcommand options will '
+                             'be loaded from a config file if they are not '
+                             'given on the command line: '
+                             'rows_per_shard, '
+                             'max_shrinking, '
+                             'max_expanding, '
+                             'shrink_threshold, '
+                             'expansion_limit')
     parser.add_argument('--verbose', '-v', action='count', default=0,
                         help='Increase output verbosity')
     # this is useful for probe tests that shard containers with unrealistically

swift/container/sharder.py

@@ -627,7 +627,9 @@ class ContainerSharderConf(object):
             'shrink_threshold', int, self.shrink_threshold)
         self.expansion_limit = get_val(
             'expansion_limit', int, self.expansion_limit)
-        self.rows_per_shard = self.shard_container_threshold // 2
+        self.rows_per_shard = get_val(
+            'rows_per_shard', config_positive_int_value,
+            max(self.shard_container_threshold // 2, 1))
 
     def percent_of_threshold(self, val):
         return int(config_percent_value(val) * self.shard_container_threshold)

test/unit/cli/test_manage_shard_ranges.py

@@ -139,6 +139,7 @@ class TestManageShardRanges(unittest.TestCase):
         shrink_threshold = 150
         expansion_limit = 650
         shard_container_threshold = 1000
+        rows_per_shard = 600
         max_shrinking = 33
         max_expanding = 31
         """
@@ -169,7 +170,7 @@ class TestManageShardRanges(unittest.TestCase):
         expected = Namespace(conf_file=conf_file,
                              path_to_file=mock.ANY,
                              func=mock.ANY,
-                             rows_per_shard=500,
+                             rows_per_shard=600,
                              subcommand='find',
                              force_commits=False,
                              verbose=0)

test/unit/container/test_sharder.py

@@ -186,6 +186,20 @@ class TestSharder(BaseTestSharder):
             allow_modify_pipeline=False,
             use_replication_network=True)
 
+        # non-default shard_container_threshold influences other defaults
+        conf = {'shard_container_threshold': 20000000}
+        expected.update({
+            'shard_container_threshold': 20000000,
+            'shrink_threshold': 2000000,
+            'expansion_limit': 15000000,
+            'rows_per_shard': 10000000
+        })
+        sharder, mock_ic = self._do_test_init(conf, expected)
+        mock_ic.assert_called_once_with(
+            '/etc/swift/internal-client.conf', 'Swift Container Sharder', 3,
+            allow_modify_pipeline=False,
+            use_replication_network=True)
+
         # non-default values
         conf = {
             'mount_check': False, 'bind_ip': '10.11.12.13', 'bind_port': 62010,
@@ -212,7 +226,7 @@ class TestSharder(BaseTestSharder):
             'existing_shard_replication_quorum': 0,
             'max_shrinking': 5,
             'max_expanding': 4,
-            'rows_per_shard': 13,  # should be ignored - not configurable
+            'rows_per_shard': 13
         }
         expected = {
             'mount_check': False, 'bind_ip': '10.11.12.13', 'port': 62010,
@@ -224,7 +238,7 @@ class TestSharder(BaseTestSharder):
             'rsync_module': '{replication_ip}::container_sda',
             'reclaim_age': 86400 * 14,
             'shard_container_threshold': 20000000,
-            'rows_per_shard': 10000000,
+            'rows_per_shard': 13,
             'shrink_threshold': 7000000,
             'expansion_limit': 17000000,
             'cleave_batch_size': 4,
@@ -7293,8 +7307,7 @@ class TestContainerSharderConf(unittest.TestCase):
                 'shrink_threshold': 100001,
                 'expansion_limit': 750001,
                 'rows_per_shard': 500001}
-        # rows_per_shard is not directly configurable
+        expected = dict(conf)
-        expected = dict(conf, rows_per_shard=1000000)
         conf.update({'unexpected': 'option'})
         self.assertEqual(expected, vars(ContainerSharderConf(conf)))