openstack/swift
Code Issues Proposed changes

simplifying options and code

Browse Source
This commit is contained in:
David Goetz 2011-02-21 16:37:12 -08:00
parent 7144693bf5
commit a86a569cae
5 changed files with 35 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
bin/swift-object-auditor
View File

@ -21,9 +21,7 @@ from optparse import OptionParser
if __name__ == '__main__':
parser = OptionParser("%prog CONFIG [options]")
parser.add_option('-z', '--zero_byte_only', default=False,
action='store_true', help='Audit only zero byte files')
parser.add_option('-f', '--zero_byte_fps',
help='Override zero byte files per second in config.')
parser.add_option('-z', '--zero_byte_fps',
help='Audit only zero byte files at specified files/sec')
conf_file, options = parse_options(parser=parser, once=True)
run_daemon(ObjectAuditor, conf_file, **options)

7
doc/source/admin_guide.rst
View File

@ -297,7 +297,6 @@ trying to write and produce zero byte files. The object-auditor will catch
these problems but in the case of a system crash it would be advisable to run
an extra, less rate limited sweep to check for these specific files. You can
run this command as follows:
`swift-object-auditor /path/to/object-server/config/file.conf once -z -f 1000`
"-z" will check for only zero-byte files and "-f" overrides the
zero_byte_files_per_second to a 1000 from the config file, which by default is
only 50.
`swift-object-auditor /path/to/object-server/config/file.conf once -z 1000`
"-z" means to only check for zero-byte files at 1000 files per second.

2
etc/object-server.conf-sample
View File

@ -72,3 +72,5 @@ use = egg:swift#object
# files_per_second = 20
# bytes_per_second = 10000000
# log_time = 3600
# zero_byte_files_per_second = 50

41
swift/obj/auditor.py
View File

@ -29,7 +29,7 @@ from swift.common.daemon import Daemon
class AuditorWorker(object):
"""Walk through file system to audit object"""
def __init__(self, conf, zero_byte_file_worker=False, zero_byte_fps=None):
def __init__(self, conf, zero_byte_only_at_fps=0):
self.conf = conf
self.logger = get_logger(conf, log_route='object-auditor')
self.devices = conf.get('devices', '/srv/node')
@ -39,16 +39,9 @@ class AuditorWorker(object):
self.max_bytes_per_second = float(conf.get('bytes_per_second',
10000000))
self.auditor_type = 'ALL'
self.fasttrack_zero_byte_files = conf.get(
'fasttrack_zero_byte_files', 'False').lower() in TRUE_VALUES
self.zero_byte_file_worker = zero_byte_file_worker
if self.zero_byte_file_worker:
self.fasttrack_zero_byte_files = True
if zero_byte_fps:
self.max_files_per_second = float(zero_byte_fps)
else:
self.max_files_per_second = float(
conf.get('zero_byte_files_per_second', 50))
self.zero_byte_only_at_fps = zero_byte_only_at_fps
if self.zero_byte_only_at_fps:
self.max_files_per_second = float(self.zero_byte_only_at_fps)
self.auditor_type = 'ZBF'
self.log_time = int(conf.get('log_time', 3600))
self.files_running_time = 0
@ -135,8 +128,7 @@ class AuditorWorker(object):
raise AuditException('Content-Length of %s does not match '
'file size of %s' % (int(df.metadata['Content-Length']),
os.path.getsize(df.data_file)))
if self.fasttrack_zero_byte_files and \
bool(self.zero_byte_file_worker) == bool(obj_size):
if self.zero_byte_only_at_fps and obj_size:
return
etag = md5()
for chunk in df:
@ -172,31 +164,30 @@ class ObjectAuditor(Daemon):
def __init__(self, conf, **options):
self.conf = conf
self.logger = get_logger(conf, 'object-auditor')
self.fasttrack_zero_byte_files = conf.get(
'fasttrack_zero_byte_files', 'False').lower() in TRUE_VALUES
self.conf_zero_byte_fps = int(conf.get(
'zero_byte_files_per_second', 50))
def run_forever(self, *args, **kwargs):
"""Run the object audit until stopped."""
zero_byte_only = kwargs.get('zero_byte_only', False)
zero_byte_fps = kwargs.get('zero_byte_fps', None)
zero_byte_only_at_fps = kwargs.get('zero_byte_fps', 0) or \
self.conf_zero_byte_fps
zero_byte_pid = 1
if zero_byte_only or self.fasttrack_zero_byte_files:
if zero_byte_only_at_fps:
zero_byte_pid = os.fork()
if zero_byte_pid == 0:
while True:
self.run_once(mode='forever', zero_byte_only=True,
zero_byte_fps=zero_byte_fps)
self.run_once(mode='forever',
zero_byte_fps=zero_byte_only_at_fps)
time.sleep(30)
else:
while not zero_byte_only:
while not zero_byte_only_at_fps:
self.run_once(mode='forever')
time.sleep(30)
def run_once(self, *args, **kwargs):
"""Run the object audit once."""
mode = kwargs.get('mode', 'once')
zero_byte_only = kwargs.get('zero_byte_only', False)
zero_byte_fps = kwargs.get('zero_byte_fps', None)
worker = AuditorWorker(self.conf, zero_byte_file_worker=zero_byte_only,
zero_byte_fps=zero_byte_fps)
zero_byte_only_at_fps = kwargs.get('zero_byte_fps', 0)
worker = AuditorWorker(self.conf,
zero_byte_only_at_fps=zero_byte_only_at_fps)
worker.audit_all_objects(mode=mode)

17
test/unit/obj/test_auditor.py
View File

@ -234,7 +234,6 @@ class TestAuditor(unittest.TestCase):
self.assertEquals(self.auditor.quarantines, pre_quarantines + 1)
def test_object_run_fast_track_non_zero(self):
self.conf['fasttrack_zero_byte_files'] = 'yes'
self.auditor = auditor.ObjectAuditor(self.conf)
self.auditor.log_time = 0
cur_part = '0'
@ -259,13 +258,12 @@ class TestAuditor(unittest.TestCase):
quarantine_path = os.path.join(self.devices,
'sda', 'quarantined', 'objects')
self.auditor.run_once(zero_byte_only=True)
self.auditor.run_once(zero_byte_fps=50)
self.assertFalse(os.path.isdir(quarantine_path))
self.auditor.run_once()
self.assertTrue(os.path.isdir(quarantine_path))
def test_object_run_fast_track_zero(self):
self.conf['fasttrack_zero_byte_files'] = 'yes'
def setup_bad_zero_byte(self):
self.auditor = auditor.ObjectAuditor(self.conf)
self.auditor.log_time = 0
cur_part = '0'
@ -283,11 +281,20 @@ class TestAuditor(unittest.TestCase):
etag = etag.hexdigest()
metadata['ETag'] = etag
write_metadata(fd, metadata)
def test_object_run_fast_track_all(self):
self.setup_bad_zero_byte()
self.auditor.run_once()
quarantine_path = os.path.join(self.devices,
'sda', 'quarantined', 'objects')
self.auditor.run_once()
self.assertTrue(os.path.isdir(quarantine_path))
def test_object_run_fast_track_zero(self):
self.setup_bad_zero_byte()
self.auditor.run_once(zero_byte_fps=50)
quarantine_path = os.path.join(self.devices,
'sda', 'quarantined', 'objects')
self.assertTrue(os.path.isdir(quarantine_path))
if __name__ == '__main__':
unittest.main()