relinker: Add option to ratelimit relinking

Sure, you could use stuff like ionice or cgroups to limit relinker I/O,
but sometimes a nice simple blunt instrument is handy.

Change-Id: I7fe29c7913a9e09bdf7a787ccad8bba2c77cf995

etc/object-server.conf-sample

@@ -609,3 +609,8 @@ use = egg:swift#xprofile
 # log_facility = LOG_LOCAL0
 # log_level = INFO
 # log_address = /dev/log
+#
+# Target this many relinks/cleanups per second, to reduce the
+# likelihood that the added I/O from a partition-power increase impacts
+# client traffic. Use zero for unlimited.
+# files_per_second = 0.0

swift/cli/relinker.py

@@ -25,7 +25,8 @@ from swift.common.storage_policy import POLICIES
 from swift.common.exceptions import DiskFileDeleted, DiskFileNotExist, \
     DiskFileQuarantined
 from swift.common.utils import replace_partition_in_path, config_true_value, \
-    audit_location_generator, get_logger, readconf, drop_privileges
+    audit_location_generator, get_logger, readconf, drop_privileges, \
+    RateLimitedIterator
 from swift.obj import diskfile
 
 
@@ -36,6 +37,13 @@ STEP_RELINK = 'relink'
 STEP_CLEANUP = 'cleanup'
 
 
+def non_negative_float(value):
+    value = float(value)
+    if value < 0:
+        raise ValueError
+    return value
+
+
 def devices_filter(device, _, devices):
     if device:
         devices = [d for d in devices if d == device]
@@ -157,7 +165,8 @@ def relink(swift_dir='/etc/swift',
            devices='/srv/node',
            skip_mount_check=False,
            logger=logging.getLogger(),
-           device=None):
+           device=None,
+           files_per_second=0):
     mount_check = not skip_mount_check
     run = False
     relinked = errors = 0
@@ -200,6 +209,8 @@ def relink(swift_dir='/etc/swift',
             hook_post_partition=relink_hook_post_partition,
             hashes_filter=relink_hashes_filter,
             logger=logger)
+        if files_per_second > 0:
+            locations = RateLimitedIterator(locations, files_per_second)
         for fname, _, _ in locations:
             newfname = replace_partition_in_path(fname, next_part_power)
             try:
@@ -223,7 +234,8 @@ def cleanup(swift_dir='/etc/swift',
             devices='/srv/node',
             skip_mount_check=False,
             logger=logging.getLogger(),
-            device=None):
+            device=None,
+            files_per_second=0):
     mount_check = not skip_mount_check
     conf = {'devices': devices, 'mount_check': mount_check}
     diskfile_router = diskfile.DiskFileRouter(conf, logger)
@@ -268,6 +280,8 @@ def cleanup(swift_dir='/etc/swift',
             hook_post_partition=cleanup_hook_post_partition,
             hashes_filter=cleanup_hashes_filter,
             logger=logger)
+        if files_per_second > 0:
+            locations = RateLimitedIterator(locations, files_per_second)
         for fname, device, partition in locations:
             expected_fname = replace_partition_in_path(fname, part_power)
             if fname == expected_fname:
@@ -340,6 +354,9 @@ def main(args):
     parser.add_argument('--skip-mount-check', default=False,
                         help='Don\'t test if disk is mounted',
                         action="store_true", dest='skip_mount_check')
+    parser.add_argument('--files-per-second', default=None,
+                        type=non_negative_float, dest='files_per_second',
+                        help='Used to limit I/O (default: no limit)')
     parser.add_argument('--logfile', default=None, dest='logfile',
                         help='Set log file name. Ignored if using conf_file.')
     parser.add_argument('--debug', default=False, action='store_true',
@@ -369,11 +386,15 @@ def main(args):
     devices = args.devices or conf.get('devices', '/srv/node')
     skip_mount_check = args.skip_mount_check or not config_true_value(
         conf.get('mount_check', 'true'))
+    files_per_second = non_negative_float(
+        args.files_per_second or conf.get('files_per_second', '0'))
 
     if args.action == 'relink':
         return relink(
-            swift_dir, devices, skip_mount_check, logger, device=args.device)
+            swift_dir, devices, skip_mount_check, logger, device=args.device,
+            files_per_second=files_per_second)
 
     if args.action == 'cleanup':
         return cleanup(
-            swift_dir, devices, skip_mount_check, logger, device=args.device)
+            swift_dir, devices, skip_mount_check, logger, device=args.device,
+            files_per_second=files_per_second)

test/unit/cli/test_relinker.py

@@ -163,7 +163,8 @@ class TestRelinker(unittest.TestCase):
         with mock.patch('swift.cli.relinker.relink') as mock_relink:
             relinker.main(['relink', conf_file, '--device', 'sdx', '--debug'])
         mock_relink.assert_called_once_with(
-            'test/swift/dir', '/test/node', True, mock.ANY, device='sdx')
+            'test/swift/dir', '/test/node', True, mock.ANY, device='sdx',
+            files_per_second=0.0)
         logger = mock_relink.call_args[0][3]
         # --debug overrides conf file
         self.assertEqual(logging.DEBUG, logger.getEffectiveLevel())
@@ -179,24 +180,28 @@ class TestRelinker(unittest.TestCase):
         [object-relinker]
         log_level = WARNING
         log_name = test-relinker
+        files_per_second = 11.1
         """
         with open(conf_file, 'w') as f:
             f.write(dedent(config))
         with mock.patch('swift.cli.relinker.relink') as mock_relink:
             relinker.main(['relink', conf_file, '--device', 'sdx'])
         mock_relink.assert_called_once_with(
-            'test/swift/dir', '/test/node', False, mock.ANY, device='sdx')
+            'test/swift/dir', '/test/node', False, mock.ANY, device='sdx',
+            files_per_second=11.1)
         logger = mock_relink.call_args[0][3]
         self.assertEqual(logging.WARNING, logger.getEffectiveLevel())
         self.assertEqual('test-relinker', logger.logger.name)
 
         # override with cli options...
         with mock.patch('swift.cli.relinker.relink') as mock_relink:
-            relinker.main(['relink', conf_file, '--device', 'sdx', '--debug',
+            relinker.main([
+                'relink', conf_file, '--device', 'sdx', '--debug',
                 '--swift-dir', 'cli-dir', '--devices', 'cli-devs',
-                           '--skip-mount-check'])
+                '--skip-mount-check', '--files-per-second', '2.2'])
         mock_relink.assert_called_once_with(
-            'cli-dir', 'cli-devs', True, mock.ANY, device='sdx')
+            'cli-dir', 'cli-devs', True, mock.ANY, device='sdx',
+            files_per_second=2.2)
 
         with mock.patch('swift.cli.relinker.relink') as mock_relink, \
                 mock.patch('logging.basicConfig') as mock_logging_config:
@@ -204,7 +209,8 @@ class TestRelinker(unittest.TestCase):
                            '--swift-dir', 'cli-dir', '--devices', 'cli-devs',
                            '--skip-mount-check'])
         mock_relink.assert_called_once_with(
-            'cli-dir', 'cli-devs', True, mock.ANY, device='sdx')
+            'cli-dir', 'cli-devs', True, mock.ANY, device='sdx',
+            files_per_second=0.0)
         mock_logging_config.assert_called_once_with(
             format='%(message)s', level=logging.INFO, filename=None)
 
@@ -214,7 +220,8 @@ class TestRelinker(unittest.TestCase):
                            '--swift-dir', 'cli-dir', '--devices', 'cli-devs',
                            '--skip-mount-check'])
         mock_relink.assert_called_once_with(
-            'cli-dir', 'cli-devs', True, mock.ANY, device='sdx')
+            'cli-dir', 'cli-devs', True, mock.ANY, device='sdx',
+            files_per_second=0.0)
         # --debug is now effective
         mock_logging_config.assert_called_once_with(
             format='%(message)s', level=logging.DEBUG, filename=None)