openstack/swift
Code Issues Proposed changes

py3: Fix swift-drive-audit

Browse Source 
Walking through the kernel logs backwards requires that we open them
in binary mode. Add a new option to allow users to specify which
encoding should be used to interpret those logs; default to the same
encoding that open() uses for its default.

Change-Id: Iae332bb58388b5521445e75beba6ee2e9f06bfa6
Closes-Bug: #1847955
This commit is contained in:
Tim Burke 2019-10-13 21:55:54 -07:00
parent 29d46ca9f6
commit 405a2b2a55
2 changed files with 17 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
bin/swift-drive-audit
View File

@ -16,12 +16,14 @@
import datetime import datetime
import glob import glob
import locale
import os import os
import re import re
import subprocess import subprocess
import sys import sys
import six
from six.moves.configparser import ConfigParser from six.moves.configparser import ConfigParser
from swift.common.utils import backward, get_logger, dump_recon_cache, \ from swift.common.utils import backward, get_logger, dump_recon_cache, \
@ -57,7 +59,8 @@ def get_devices(device_dir, logger):
return devices return devices
def get_errors(error_re, log_file_pattern, minutes, logger): def get_errors(error_re, log_file_pattern, minutes, logger,
log_file_encoding):
# Assuming log rotation is being used, we need to examine # Assuming log rotation is being used, we need to examine
# recently rotated files in case the rotation occurred # recently rotated files in case the rotation occurred
# just before the script is being run - the data we are # just before the script is being run - the data we are
@ -87,12 +90,14 @@ def get_errors(error_re, log_file_pattern, minutes, logger):
reached_old_logs = False reached_old_logs = False
for path in log_files: for path in log_files:
try: try:
f = open(path) f = open(path, 'rb')
except IOError: except IOError:
logger.error("Error: Unable to open " + path) logger.error("Error: Unable to open " + path)
print("Unable to open " + path) print("Unable to open " + path)
sys.exit(1) sys.exit(1)
for line in backward(f): for line in backward(f):
if not six.PY2:
line = line.decode(log_file_encoding, 'surrogateescape')
if '[ 0.000000]' in line \ if '[ 0.000000]' in line \
or 'KERNEL supported cpus:' in line \ or 'KERNEL supported cpus:' in line \
or 'BIOS-provided physical RAM map:' in line: or 'BIOS-provided physical RAM map:' in line:
@ -105,7 +110,7 @@ def get_errors(error_re, log_file_pattern, minutes, logger):
if log_time_entry[0] == 'Dec' and prev_entry_month == 'Jan': if log_time_entry[0] == 'Dec' and prev_entry_month == 'Jan':
year -= 1 year -= 1
prev_entry_month = log_time_entry[0] prev_entry_month = log_time_entry[0]
log_time_string = '%s %s' % (year, ' '.join(log_time_entry)) log_time_string = '%d %s' % (year, ' '.join(log_time_entry))
try: try:
log_time = datetime.datetime.strptime( log_time = datetime.datetime.strptime(
log_time_string, '%Y %b %d %H:%M:%S') log_time_string, '%Y %b %d %H:%M:%S')
@ -154,6 +159,9 @@ if __name__ == '__main__':
recon_cache_path = conf.get('recon_cache_path', "/var/cache/swift") recon_cache_path = conf.get('recon_cache_path', "/var/cache/swift")
log_file_pattern = conf.get('log_file_pattern', log_file_pattern = conf.get('log_file_pattern',
'/var/log/kern.*[!.][!g][!z]') '/var/log/kern.*[!.][!g][!z]')
log_file_encoding = conf.get('log_file_encoding', 'auto')
if log_file_encoding == 'auto':
log_file_encoding = locale.getpreferredencoding()
log_to_console = config_true_value(conf.get('log_to_console', False)) log_to_console = config_true_value(conf.get('log_to_console', False))
error_re = [] error_re = []
for conf_key in conf: for conf_key in conf:
@ -181,7 +189,8 @@ if __name__ == '__main__':
total_errors = 0 total_errors = 0
for device in devices: for device in devices:
recon_errors[device['mount_point']] = 0 recon_errors[device['mount_point']] = 0
errors = get_errors(error_re, log_file_pattern, minutes, logger) errors = get_errors(error_re, log_file_pattern, minutes, logger,
log_file_encoding)
logger.debug("Errors found: %s" % str(errors)) logger.debug("Errors found: %s" % str(errors))
unmounts = 0 unmounts = 0
for kernel_device, count in errors.items(): for kernel_device, count in errors.items():

4
etc/drive-audit.conf-sample
View File

@ -26,6 +26,10 @@
# pattern to check against device errors. # pattern to check against device errors.
# log_file_pattern = /var/log/kern.*[!.][!g][!z] # log_file_pattern = /var/log/kern.*[!.][!g][!z]
# #
# On Python 3, the encoding to use when reading the log file. Defaults
# to the result of locale.getpreferredencoding(), like Python's open().
# log_file_encoding = auto
#
# Regular expression patterns to be used to locate # Regular expression patterns to be used to locate
# device blocks with errors in the log file. Currently # device blocks with errors in the log file. Currently
# the default ones are as follows: # the default ones are as follows: