Merge "DB Replicator: Add handoff_delete option"

doc/source/config/account_server_config.rst

@@ -170,9 +170,9 @@ ionice_priority                None            I/O scheduling priority of server
 [account-replicator]
 ********************
 
-==================== =========================  ===============================
+==================== =========================  =====================================
 Option               Default                    Description
--------------------- -------------------------  -------------------------------
+-------------------- -------------------------  -------------------------------------
 log_name             account-replicator         Label used when logging
 log_facility         LOG_LOCAL0                 Syslog log facility
 log_level            INFO                       Logging level
@@ -256,7 +256,26 @@ ionice_priority      None                       I/O scheduling priority of serve
                                                 Work only with ionice_class.
                                                 Ignored if IOPRIO_CLASS_IDLE
                                                 is set.
-==================== =========================  ===============================
+handoffs_only        no                         When handoffs_only mode is enabled
+                                                the replicator will *only* replicate
+                                                from handoff nodes to primary nodes
+                                                and will not sync primary nodes
+                                                with other primary nodes.
+handoff_delete       auto                       the number of replicas which are
+                                                ensured in swift. If the number
+                                                less than the number of replicas
+                                                is set, account-replicator
+                                                could delete local handoffs even
+                                                if all replicas are not ensured in
+                                                the cluster. The replicator would
+                                                remove local handoff account database
+                                                after syncing when the number of
+                                                successful responses is greater than
+                                                or equal to this number. By default
+                                                handoff partitions will be removed
+                                                when it has successfully replicated
+                                                to all the canonical nodes.
+==================== =========================  =====================================
 
 *****************
 [account-auditor]

doc/source/config/container_server_config.rst

@@ -175,9 +175,9 @@ ionice_priority                 None              I/O scheduling priority of ser
 [container-replicator]
 **********************
 
-==================== ===========================  =============================
+==================== ===========================  =======================================
 Option               Default                      Description
--------------------- ---------------------------  -----------------------------
+-------------------- ---------------------------  ---------------------------------------
 log_name             container-replicator         Label used when logging
 log_facility         LOG_LOCAL0                   Syslog log facility
 log_level            INFO                         Logging level
@@ -266,7 +266,26 @@ ionice_priority      None                         I/O scheduling priority of
                                                   Work only with ionice_class.
                                                   Ignored if IOPRIO_CLASS_IDLE
                                                   is set.
-==================== ===========================  =============================
+handoffs_only        no                           When handoffs_only mode is enabled
+                                                  the replicator will *only* replicate
+                                                  from handoff nodes to primary nodes
+                                                  and will not sync primary nodes
+                                                  with other primary nodes.
+handoff_delete       auto                         the number of replicas which are
+                                                  ensured in swift. If the number
+                                                  less than the number of replicas
+                                                  is set, container-replicator
+                                                  could delete local handoffs even
+                                                  if all replicas are not ensured in
+                                                  the cluster. The replicator would
+                                                  remove local handoff container database
+                                                  after syncing when the number of
+                                                  successful responses is greater than
+                                                  or equal to this number. By default
+                                                  handoff partitions will be removed
+                                                  when it has successfully replicated
+                                                  to all the canonical nodes.
+==================== ===========================  =======================================
 
 *******************
 [container-sharder]

etc/account-server.conf-sample

@@ -187,8 +187,8 @@ use = egg:swift#recon
 # ionice_class =
 # ionice_priority =
 #
-# The handoffs_only mode option is for special-case emergency
+# The handoffs_only and handoff_delete options are for special-case emergency
-# situations such as full disks in the cluster. This option SHOULD NOT
+# situations such as full disks in the cluster. These options SHOULD NOT
 # BE ENABLED except in emergencies. When handoffs_only mode is enabled
 # the replicator will *only* replicate from handoff nodes to primary
 # nodes and will not sync primary nodes with other primary nodes.
@@ -205,6 +205,15 @@ use = egg:swift#recon
 # long-term use.
 #
 # handoffs_only = no
+#
+# handoff_delete is the number of replicas which are ensured in swift.
+# If the number less than the number of replicas is set, account-replicator
+# could delete local handoffs even if all replicas are not ensured in the
+# cluster. The replicator would remove local handoff account database after
+# syncing when the number of successful responses is greater than or equal to
+# this number. By default(auto), handoff partitions will be
+# removed  when it has successfully replicated to all the canonical nodes.
+# handoff_delete = auto
 
 [account-auditor]
 # You can override the default log routing for this app here (don't use set!):

etc/container-server.conf-sample

@@ -197,8 +197,8 @@ use = egg:swift#recon
 # ionice_class =
 # ionice_priority =
 #
-# The handoffs_only mode option is for special-case emergency
+# The handoffs_only and handoff_delete options are for special-case emergency
-# situations such as full disks in the cluster. This option SHOULD NOT
+# situations such as full disks in the cluster. These options SHOULD NOT
 # BE ENABLED except in emergencies. When handoffs_only mode is enabled
 # the replicator will *only* replicate from handoff nodes to primary
 # nodes and will not sync primary nodes with other primary nodes.
@@ -215,6 +215,15 @@ use = egg:swift#recon
 # long-term use.
 #
 # handoffs_only = no
+#
+# handoff_delete is the number of replicas which are ensured in swift.
+# If the number less than the number of replicas is set, container-replicator
+# could delete local handoffs even if all replicas are not ensured in the
+# cluster. The replicator would remove local handoff container database after
+# syncing when the number of successful responses is greater than or equal to
+# this number. By default(auto), handoff partitions will be
+# removed  when it has successfully replicated to all the canonical nodes.
+# handoff_delete = auto
 
 [container-updater]
 # You can override the default log routing for this app here (don't use set!):

swift/common/db_replicator.py

@@ -34,7 +34,7 @@ from swift.common.utils import get_logger, whataremyips, storage_directory, \
     renamer, mkdirs, lock_parent_directory, config_true_value, \
     unlink_older_than, dump_recon_cache, rsync_module_interpolation, \
     parse_override_options, round_robin_iter, Everything, get_db_files, \
-    parse_db_filename, quote, RateLimitedIterator
+    parse_db_filename, quote, RateLimitedIterator, config_auto_int_value
 from swift.common import ring
 from swift.common.ring.utils import is_local_device
 from swift.common.http import HTTP_NOT_FOUND, HTTP_INSUFFICIENT_STORAGE, \
@@ -239,6 +239,8 @@ class Replicator(Daemon):
         self.extract_device_re = re.compile('%s%s([^%s]+)' % (
             self.root, os.path.sep, os.path.sep))
         self.handoffs_only = config_true_value(conf.get('handoffs_only', 'no'))
+        self.handoff_delete = config_auto_int_value(
+            conf.get('handoff_delete', 'auto'), 0)
 
     def _zero_stats(self):
         """Zero out the stats."""
@@ -555,7 +557,13 @@ class Replicator(Daemon):
             reason = '%s new rows' % max_row_delta
             self.logger.debug(log_template, reason)
             return True
-        if not (responses and all(responses)):
+        if self.handoff_delete:
+            # delete handoff if we have had handoff_delete successes
+            successes_count = len([resp for resp in responses if resp])
+            delete_handoff = successes_count >= self.handoff_delete
+        else:
+            delete_handoff = responses and all(responses)
+        if not delete_handoff:
             reason = '%s/%s success' % (responses.count(True), len(responses))
             self.logger.debug(log_template, reason)
             return True
@@ -780,11 +788,12 @@ class Replicator(Daemon):
             self.logger.error(_('ERROR Failed to get my own IPs?'))
             return
 
-        if self.handoffs_only:
+        if self.handoffs_only or self.handoff_delete:
             self.logger.warning(
-                'Starting replication pass with handoffs_only enabled. '
+                'Starting replication pass with handoffs_only '
-                'This mode is not intended for normal '
+                'and/or handoffs_delete enabled. '
-                'operation; use handoffs_only with care.')
+                'These modes are not intended for normal '
+                'operation; use these options with care.')
 
         self._local_device_ids = set()
         found_local = False
@@ -827,10 +836,11 @@ class Replicator(Daemon):
                 self._replicate_object, part, object_file, node_id)
         self.cpool.waitall()
         self.logger.info(_('Replication run OVER'))
-        if self.handoffs_only:
+        if self.handoffs_only or self.handoff_delete:
             self.logger.warning(
-                'Finished replication pass with handoffs_only enabled. '
+                'Finished replication pass with handoffs_only and/or '
-                'If handoffs_only is no longer required, disable it.')
+                'handoffs_delete enabled. If these are no longer required, '
+                'disable them.')
         self._report_stats()
 
     def run_forever(self, *args, **kwargs):

test/unit/common/test_db_replicator.py

@@ -847,6 +847,51 @@ class TestDBReplicator(unittest.TestCase):
         self.assertEqual(['/path/to/file'], self.delete_db_calls)
         self.assertEqual(0, replicator.stats['failure'])
 
+    def test_handoff_delete(self):
+        def do_test(config, repl_to_node_results, expect_delete):
+            self.delete_db_calls = []
+            replicator = TestReplicator(config)
+            replicator.ring = FakeRingWithNodes().Ring('path')
+            replicator.brokerclass = FakeAccountBroker
+            mock_repl_to_node = mock.Mock()
+            mock_repl_to_node.side_effect = repl_to_node_results
+            replicator._repl_to_node = mock_repl_to_node
+            replicator.delete_db = self.stub_delete_db
+            orig_cleanup = replicator.cleanup_post_replicate
+            with mock.patch.object(replicator, 'cleanup_post_replicate',
+                                   side_effect=orig_cleanup) as mock_cleanup:
+                replicator._replicate_object('0', '/path/to/file', 'node_id')
+            mock_cleanup.assert_called_once_with(mock.ANY, mock.ANY,
+                                                 repl_to_node_results)
+            self.assertIsInstance(mock_cleanup.call_args[0][0],
+                                  replicator.brokerclass)
+            if expect_delete:
+                self.assertEqual(['/path/to/file'], self.delete_db_calls)
+            else:
+                self.assertNotEqual(['/path/to/file'], self.delete_db_calls)
+
+            self.assertEqual(repl_to_node_results.count(True),
+                             replicator.stats['success'])
+            self.assertEqual(repl_to_node_results.count(False),
+                             replicator.stats['failure'])
+
+        for cfg, repl_results, expected_delete in (
+                # Start with the sanilty check
+                ({}, [True] * 3, True),
+                ({}, [True, True, False], False),
+                ({'handoff_delete': 'auto'}, [True] * 3, True),
+                ({'handoff_delete': 'auto'}, [True, True, False], False),
+                ({'handoff_delete': 0}, [True] * 3, True),
+                ({'handoff_delete': 0}, [True, True, False], False),
+                # Now test a lower handoff delete
+                ({'handoff_delete': 2}, [True] * 3, True),
+                ({'handoff_delete': 2}, [True, True, False], True),
+                ({'handoff_delete': 2}, [True, False, False], False),
+                ({'handoff_delete': 1}, [True] * 3, True),
+                ({'handoff_delete': 1}, [True, True, False], True),
+                ({'handoff_delete': 1}, [True, False, False], True)):
+            do_test(cfg, repl_results, expected_delete)
+
     def test_replicate_object_delete_delegated_to_cleanup_post_replicate(self):
         replicator = TestReplicator({})
         replicator.ring = FakeRingWithNodes().Ring('path')
@@ -1860,11 +1905,13 @@ class TestHandoffsOnly(unittest.TestCase):
 
         self.assertEqual(
             self.logger.get_lines_for_level('warning'),
-            [('Starting replication pass with handoffs_only enabled. This '
+            [('Starting replication pass with handoffs_only and/or '
-              'mode is not intended for normal operation; use '
+              'handoffs_delete enabled. These '
-              'handoffs_only with care.'),
+              'modes are not intended for normal operation; use '
-             ('Finished replication pass with handoffs_only enabled. '
+              'these options with care.'),
-              'If handoffs_only is no longer required, disable it.')])
+             ('Finished replication pass with handoffs_only and/or '
+              'handoffs_delete enabled. If these are no longer required, '
+              'disable them.')])
 
     def test_skips_primary_partitions(self):
         replicator = TestReplicator({