diff --git a/etc/object-server.conf-sample b/etc/object-server.conf-sample index e7466bba83..71a264fab0 100644 --- a/etc/object-server.conf-sample +++ b/etc/object-server.conf-sample @@ -291,6 +291,13 @@ use = egg:swift#recon # ring_check_interval = 15.0 # recon_cache_path = /var/cache/swift # +# By default, per-file rsync transfers are logged at debug if successful and +# error on failure. During large rebalances (which both increase the number +# of diskfiles transferred and increases the likelihood of failures), this +# can overwhelm log aggregation while providing little useful insights. +# Change this to false to disable per-file logging. +# log_rsync_transfers = true +# # limits how long rsync error log lines are # 0 means to log the entire line # rsync_error_log_line_length = 0 diff --git a/swift/obj/replicator.py b/swift/obj/replicator.py index 6e84c05dc5..38bc6bd8b0 100644 --- a/swift/obj/replicator.py +++ b/swift/obj/replicator.py @@ -182,6 +182,8 @@ class ObjectReplicator(Daemon): self.default_headers = { 'Content-Length': '0', 'user-agent': 'object-replicator %s' % os.getpid()} + self.log_rsync_transfers = config_true_value( + conf.get('log_rsync_transfers', True)) self.rsync_error_log_line_length = \ int(conf.get('rsync_error_log_line_length', 0)) self.handoffs_first = config_true_value(conf.get('handoffs_first', @@ -413,8 +415,10 @@ class ObjectReplicator(Daemon): continue if result.startswith('cd+'): continue + if result.startswith('<') and not self.log_rsync_transfers: + continue if not ret_val: - self.logger.info(result) + self.logger.debug(result) else: self.logger.error(result) if ret_val: @@ -426,7 +430,8 @@ class ObjectReplicator(Daemon): log_method = self.logger.info if results else self.logger.debug log_method( "Successful rsync of %(src)s to %(dst)s (%(time).03f)", - {'src': args[-2], 'dst': args[-1], 'time': total_time}) + {'src': args[-2][:-3] + '...', 'dst': args[-1], + 'time': total_time}) return ret_val def rsync(self, node, job, suffixes): diff --git a/test/unit/obj/test_replicator.py b/test/unit/obj/test_replicator.py index d25c25490c..494d430b0e 100644 --- a/test/unit/obj/test_replicator.py +++ b/test/unit/obj/test_replicator.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import collections +import io import json import unittest import os @@ -2096,6 +2097,157 @@ class TestObjectReplicator(unittest.TestCase): self.assertEqual(expected_reqs, [ (r['method'], r['ip'], r['path']) for r in request_log.requests]) + def test_rsync_failure_logging(self): + with mock.patch('swift.obj.replicator.subprocess.Popen') as mock_popen: + mock_popen.return_value.stdout = io.BytesIO(b'\n'.join([ + b'', + b'cd+++++++++ suf', + b'cd+++++++++ suf/hash1', + b'