Merge "Fix object auditor recon and logging"
This commit is contained in:
commit
39b03d0ddd
@ -48,6 +48,7 @@ class AuditorWorker(object):
|
|||||||
self.max_files_per_second = float(self.zero_byte_only_at_fps)
|
self.max_files_per_second = float(self.zero_byte_only_at_fps)
|
||||||
self.auditor_type = 'ZBF'
|
self.auditor_type = 'ZBF'
|
||||||
self.log_time = int(conf.get('log_time', 3600))
|
self.log_time = int(conf.get('log_time', 3600))
|
||||||
|
self.last_logged = 0
|
||||||
self.files_running_time = 0
|
self.files_running_time = 0
|
||||||
self.bytes_running_time = 0
|
self.bytes_running_time = 0
|
||||||
self.bytes_processed = 0
|
self.bytes_processed = 0
|
||||||
@ -95,7 +96,7 @@ class AuditorWorker(object):
|
|||||||
self.files_running_time, self.max_files_per_second)
|
self.files_running_time, self.max_files_per_second)
|
||||||
self.total_files_processed += 1
|
self.total_files_processed += 1
|
||||||
now = time.time()
|
now = time.time()
|
||||||
if now - reported >= self.log_time:
|
if now - self.last_logged >= self.log_time:
|
||||||
self.logger.info(_(
|
self.logger.info(_(
|
||||||
'Object audit (%(type)s). '
|
'Object audit (%(type)s). '
|
||||||
'Since %(start_time)s: Locally: %(passes)d passed, '
|
'Since %(start_time)s: Locally: %(passes)d passed, '
|
||||||
@ -126,6 +127,7 @@ class AuditorWorker(object):
|
|||||||
self.quarantines = 0
|
self.quarantines = 0
|
||||||
self.errors = 0
|
self.errors = 0
|
||||||
self.bytes_processed = 0
|
self.bytes_processed = 0
|
||||||
|
self.last_logged = now
|
||||||
time_auditing += (now - loop_time)
|
time_auditing += (now - loop_time)
|
||||||
# Avoid divide by zero during very short runs
|
# Avoid divide by zero during very short runs
|
||||||
elapsed = (time.time() - begin) or 0.000001
|
elapsed = (time.time() - begin) or 0.000001
|
||||||
@ -142,12 +144,6 @@ class AuditorWorker(object):
|
|||||||
'frate': self.total_files_processed / elapsed,
|
'frate': self.total_files_processed / elapsed,
|
||||||
'brate': self.total_bytes_processed / elapsed,
|
'brate': self.total_bytes_processed / elapsed,
|
||||||
'audit': time_auditing, 'audit_rate': time_auditing / elapsed})
|
'audit': time_auditing, 'audit_rate': time_auditing / elapsed})
|
||||||
# Clear recon cache entry if device_dirs is set
|
|
||||||
if device_dirs:
|
|
||||||
cache_entry = self.create_recon_nested_dict(
|
|
||||||
'object_auditor_stats_%s' % (self.auditor_type),
|
|
||||||
device_dirs, {})
|
|
||||||
dump_recon_cache(cache_entry, self.rcache, self.logger)
|
|
||||||
if self.stats_sizes:
|
if self.stats_sizes:
|
||||||
self.logger.info(
|
self.logger.info(
|
||||||
_('Object audit stats: %s') % json.dumps(self.stats_buckets))
|
_('Object audit stats: %s') % json.dumps(self.stats_buckets))
|
||||||
|
@ -224,6 +224,8 @@ class TestAuditor(unittest.TestCase):
|
|||||||
def test_generic_exception_handling(self):
|
def test_generic_exception_handling(self):
|
||||||
auditor_worker = auditor.AuditorWorker(self.conf, self.logger,
|
auditor_worker = auditor.AuditorWorker(self.conf, self.logger,
|
||||||
self.rcache, self.devices)
|
self.rcache, self.devices)
|
||||||
|
# pretend that we logged (and reset counters) just now
|
||||||
|
auditor_worker.last_logged = time.time()
|
||||||
timestamp = str(normalize_timestamp(time.time()))
|
timestamp = str(normalize_timestamp(time.time()))
|
||||||
pre_errors = auditor_worker.errors
|
pre_errors = auditor_worker.errors
|
||||||
data = '0' * 1024
|
data = '0' * 1024
|
||||||
@ -328,6 +330,8 @@ class TestAuditor(unittest.TestCase):
|
|||||||
self.rcache, self.devices)
|
self.rcache, self.devices)
|
||||||
timestamp = str(normalize_timestamp(time.time()))
|
timestamp = str(normalize_timestamp(time.time()))
|
||||||
pre_quarantines = auditor_worker.quarantines
|
pre_quarantines = auditor_worker.quarantines
|
||||||
|
# pretend that we logged (and reset counters) just now
|
||||||
|
auditor_worker.last_logged = time.time()
|
||||||
data = '0' * 1024
|
data = '0' * 1024
|
||||||
etag = md5()
|
etag = md5()
|
||||||
with self.disk_file.create() as writer:
|
with self.disk_file.create() as writer:
|
||||||
@ -347,6 +351,8 @@ class TestAuditor(unittest.TestCase):
|
|||||||
def test_object_run_once_multi_devices(self):
|
def test_object_run_once_multi_devices(self):
|
||||||
auditor_worker = auditor.AuditorWorker(self.conf, self.logger,
|
auditor_worker = auditor.AuditorWorker(self.conf, self.logger,
|
||||||
self.rcache, self.devices)
|
self.rcache, self.devices)
|
||||||
|
# pretend that we logged (and reset counters) just now
|
||||||
|
auditor_worker.last_logged = time.time()
|
||||||
timestamp = str(normalize_timestamp(time.time()))
|
timestamp = str(normalize_timestamp(time.time()))
|
||||||
pre_quarantines = auditor_worker.quarantines
|
pre_quarantines = auditor_worker.quarantines
|
||||||
data = '0' * 10
|
data = '0' * 10
|
||||||
|
Loading…
Reference in New Issue
Block a user