From a86a569cae3bad052a460e5eabcc22ced072348d Mon Sep 17 00:00:00 2001 From: David Goetz Date: Mon, 21 Feb 2011 16:37:12 -0800 Subject: [PATCH] simplifying options and code --- bin/swift-object-auditor | 6 ++--- doc/source/admin_guide.rst | 7 +++--- etc/object-server.conf-sample | 2 ++ swift/obj/auditor.py | 41 ++++++++++++++--------------------- test/unit/obj/test_auditor.py | 17 ++++++++++----- 5 files changed, 35 insertions(+), 38 deletions(-) diff --git a/bin/swift-object-auditor b/bin/swift-object-auditor index 2c1211b8d3..c01b05bf70 100755 --- a/bin/swift-object-auditor +++ b/bin/swift-object-auditor @@ -21,9 +21,7 @@ from optparse import OptionParser if __name__ == '__main__': parser = OptionParser("%prog CONFIG [options]") - parser.add_option('-z', '--zero_byte_only', default=False, - action='store_true', help='Audit only zero byte files') - parser.add_option('-f', '--zero_byte_fps', - help='Override zero byte files per second in config.') + parser.add_option('-z', '--zero_byte_fps', + help='Audit only zero byte files at specified files/sec') conf_file, options = parse_options(parser=parser, once=True) run_daemon(ObjectAuditor, conf_file, **options) diff --git a/doc/source/admin_guide.rst b/doc/source/admin_guide.rst index 890f5f0ff8..05580a143e 100644 --- a/doc/source/admin_guide.rst +++ b/doc/source/admin_guide.rst @@ -297,7 +297,6 @@ trying to write and produce zero byte files. The object-auditor will catch these problems but in the case of a system crash it would be advisable to run an extra, less rate limited sweep to check for these specific files. You can run this command as follows: -`swift-object-auditor /path/to/object-server/config/file.conf once -z -f 1000` -"-z" will check for only zero-byte files and "-f" overrides the -zero_byte_files_per_second to a 1000 from the config file, which by default is -only 50. +`swift-object-auditor /path/to/object-server/config/file.conf once -z 1000` +"-z" means to only check for zero-byte files at 1000 files per second. + diff --git a/etc/object-server.conf-sample b/etc/object-server.conf-sample index f3f7f07346..ea9c22fac3 100644 --- a/etc/object-server.conf-sample +++ b/etc/object-server.conf-sample @@ -72,3 +72,5 @@ use = egg:swift#object # files_per_second = 20 # bytes_per_second = 10000000 # log_time = 3600 +# zero_byte_files_per_second = 50 + diff --git a/swift/obj/auditor.py b/swift/obj/auditor.py index 9b95eafe27..0e5df61da4 100644 --- a/swift/obj/auditor.py +++ b/swift/obj/auditor.py @@ -29,7 +29,7 @@ from swift.common.daemon import Daemon class AuditorWorker(object): """Walk through file system to audit object""" - def __init__(self, conf, zero_byte_file_worker=False, zero_byte_fps=None): + def __init__(self, conf, zero_byte_only_at_fps=0): self.conf = conf self.logger = get_logger(conf, log_route='object-auditor') self.devices = conf.get('devices', '/srv/node') @@ -39,16 +39,9 @@ class AuditorWorker(object): self.max_bytes_per_second = float(conf.get('bytes_per_second', 10000000)) self.auditor_type = 'ALL' - self.fasttrack_zero_byte_files = conf.get( - 'fasttrack_zero_byte_files', 'False').lower() in TRUE_VALUES - self.zero_byte_file_worker = zero_byte_file_worker - if self.zero_byte_file_worker: - self.fasttrack_zero_byte_files = True - if zero_byte_fps: - self.max_files_per_second = float(zero_byte_fps) - else: - self.max_files_per_second = float( - conf.get('zero_byte_files_per_second', 50)) + self.zero_byte_only_at_fps = zero_byte_only_at_fps + if self.zero_byte_only_at_fps: + self.max_files_per_second = float(self.zero_byte_only_at_fps) self.auditor_type = 'ZBF' self.log_time = int(conf.get('log_time', 3600)) self.files_running_time = 0 @@ -135,8 +128,7 @@ class AuditorWorker(object): raise AuditException('Content-Length of %s does not match ' 'file size of %s' % (int(df.metadata['Content-Length']), os.path.getsize(df.data_file))) - if self.fasttrack_zero_byte_files and \ - bool(self.zero_byte_file_worker) == bool(obj_size): + if self.zero_byte_only_at_fps and obj_size: return etag = md5() for chunk in df: @@ -172,31 +164,30 @@ class ObjectAuditor(Daemon): def __init__(self, conf, **options): self.conf = conf self.logger = get_logger(conf, 'object-auditor') - self.fasttrack_zero_byte_files = conf.get( - 'fasttrack_zero_byte_files', 'False').lower() in TRUE_VALUES + self.conf_zero_byte_fps = int(conf.get( + 'zero_byte_files_per_second', 50)) def run_forever(self, *args, **kwargs): """Run the object audit until stopped.""" - zero_byte_only = kwargs.get('zero_byte_only', False) - zero_byte_fps = kwargs.get('zero_byte_fps', None) + zero_byte_only_at_fps = kwargs.get('zero_byte_fps', 0) or \ + self.conf_zero_byte_fps zero_byte_pid = 1 - if zero_byte_only or self.fasttrack_zero_byte_files: + if zero_byte_only_at_fps: zero_byte_pid = os.fork() if zero_byte_pid == 0: while True: - self.run_once(mode='forever', zero_byte_only=True, - zero_byte_fps=zero_byte_fps) + self.run_once(mode='forever', + zero_byte_fps=zero_byte_only_at_fps) time.sleep(30) else: - while not zero_byte_only: + while not zero_byte_only_at_fps: self.run_once(mode='forever') time.sleep(30) def run_once(self, *args, **kwargs): """Run the object audit once.""" mode = kwargs.get('mode', 'once') - zero_byte_only = kwargs.get('zero_byte_only', False) - zero_byte_fps = kwargs.get('zero_byte_fps', None) - worker = AuditorWorker(self.conf, zero_byte_file_worker=zero_byte_only, - zero_byte_fps=zero_byte_fps) + zero_byte_only_at_fps = kwargs.get('zero_byte_fps', 0) + worker = AuditorWorker(self.conf, + zero_byte_only_at_fps=zero_byte_only_at_fps) worker.audit_all_objects(mode=mode) diff --git a/test/unit/obj/test_auditor.py b/test/unit/obj/test_auditor.py index 373e92b11c..76e6bd419d 100644 --- a/test/unit/obj/test_auditor.py +++ b/test/unit/obj/test_auditor.py @@ -234,7 +234,6 @@ class TestAuditor(unittest.TestCase): self.assertEquals(self.auditor.quarantines, pre_quarantines + 1) def test_object_run_fast_track_non_zero(self): - self.conf['fasttrack_zero_byte_files'] = 'yes' self.auditor = auditor.ObjectAuditor(self.conf) self.auditor.log_time = 0 cur_part = '0' @@ -259,13 +258,12 @@ class TestAuditor(unittest.TestCase): quarantine_path = os.path.join(self.devices, 'sda', 'quarantined', 'objects') - self.auditor.run_once(zero_byte_only=True) + self.auditor.run_once(zero_byte_fps=50) self.assertFalse(os.path.isdir(quarantine_path)) self.auditor.run_once() self.assertTrue(os.path.isdir(quarantine_path)) - def test_object_run_fast_track_zero(self): - self.conf['fasttrack_zero_byte_files'] = 'yes' + def setup_bad_zero_byte(self): self.auditor = auditor.ObjectAuditor(self.conf) self.auditor.log_time = 0 cur_part = '0' @@ -283,11 +281,20 @@ class TestAuditor(unittest.TestCase): etag = etag.hexdigest() metadata['ETag'] = etag write_metadata(fd, metadata) + + def test_object_run_fast_track_all(self): + self.setup_bad_zero_byte() + self.auditor.run_once() quarantine_path = os.path.join(self.devices, 'sda', 'quarantined', 'objects') - self.auditor.run_once() self.assertTrue(os.path.isdir(quarantine_path)) + def test_object_run_fast_track_zero(self): + self.setup_bad_zero_byte() + self.auditor.run_once(zero_byte_fps=50) + quarantine_path = os.path.join(self.devices, + 'sda', 'quarantined', 'objects') + self.assertTrue(os.path.isdir(quarantine_path)) if __name__ == '__main__': unittest.main()