#!/usr/bin/env python # Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import datetime import glob import os import re import subprocess import sys from ConfigParser import ConfigParser from swift.common.utils import backward, get_logger, dump_recon_cache, \ config_true_value def get_devices(device_dir, logger): devices = [] for line in open('/proc/mounts').readlines(): data = line.strip().split() block_device = data[0] mount_point = data[1] if mount_point.startswith(device_dir): device = {} device['mount_point'] = mount_point device['block_device'] = block_device try: device_num = os.stat(block_device).st_rdev except OSError: # If we can't stat the device, then something weird is going on logger.error("Error: Could not stat %s!" % block_device) continue device['major'] = str(os.major(device_num)) device['minor'] = str(os.minor(device_num)) devices.append(device) for line in open('/proc/partitions').readlines()[2:]: major, minor, blocks, kernel_device = line.strip().split() device = [d for d in devices if d['major'] == major and d['minor'] == minor] if device: device[0]['kernel_device'] = kernel_device return devices def get_errors(error_re, log_file_pattern, minutes, logger): # Assuming log rotation is being used, we need to examine # recently rotated files in case the rotation occurred # just before the script is being run - the data we are # looking for may have rotated. # # The globbing used before would not work with all out-of-box # distro setup for logrotate and syslog therefore moving this # to the config where one can set it with the desired # globbing pattern. log_files = [f for f in glob.glob(log_file_pattern)] try: log_files.sort(key=lambda f: os.stat(f).st_mtime, reverse=True) except (IOError, OSError) as exc: logger.error(exc) print(exc) sys.exit(1) now_time = datetime.datetime.now() end_time = now_time - datetime.timedelta(minutes=minutes) # kern.log does not contain the year so we need to keep # track of the year and month in case the year recently # ticked over year = now_time.year prev_entry_month = now_time.month errors = {} reached_old_logs = False for path in log_files: try: f = open(path) except IOError: logger.error("Error: Unable to open " + path) print("Unable to open " + path) sys.exit(1) for line in backward(f): if '[ 0.000000]' in line \ or 'KERNEL supported cpus:' in line \ or 'BIOS-provided physical RAM map:' in line: # Ignore anything before the last boot reached_old_logs = True break # Solves the problem with year change - kern.log does not # keep track of the year. log_time_entry = line.split()[:3] if log_time_entry[0] == 'Dec' and prev_entry_month == 'Jan': year -= 1 prev_entry_month = log_time_entry[0] log_time_string = '%s %s' % (year, ' '.join(log_time_entry)) try: log_time = datetime.datetime.strptime( log_time_string, '%Y %b %d %H:%M:%S') except ValueError: continue if log_time > end_time: for err in error_re: for device in err.findall(line): errors[device] = errors.get(device, 0) + 1 else: reached_old_logs = True break if reached_old_logs: break return errors def comment_fstab(mount_point): with open('/etc/fstab', 'r') as fstab: with open('/etc/fstab.new', 'w') as new_fstab: for line in fstab: parts = line.split() if len(parts) > 2 \ and parts[1] == mount_point \ and not line.startswith('#'): new_fstab.write('#' + line) else: new_fstab.write(line) os.rename('/etc/fstab.new', '/etc/fstab') if __name__ == '__main__': c = ConfigParser() try: conf_path = sys.argv[1] except Exception: print "Usage: %s CONF_FILE" % sys.argv[0].split('/')[-1] sys.exit(1) if not c.read(conf_path): print "Unable to read config file %s" % conf_path sys.exit(1) conf = dict(c.items('drive-audit')) device_dir = conf.get('device_dir', '/srv/node') minutes = int(conf.get('minutes', 60)) error_limit = int(conf.get('error_limit', 1)) recon_cache_path = conf.get('recon_cache_path', "/var/cache/swift") log_file_pattern = conf.get('log_file_pattern', '/var/log/kern.*[!.][!g][!z]') log_to_console = config_true_value(conf.get('log_to_console', False)) error_re = [] for conf_key in conf: if conf_key.startswith('regex_pattern_'): error_pattern = conf[conf_key] try: r = re.compile(error_pattern) except re.error: sys.exit('Error: unable to compile regex pattern "%s"' % error_pattern) error_re.append(r) if not error_re: error_re = [ re.compile(r'\berror\b.*\b(sd[a-z]{1,2}\d?)\b'), re.compile(r'\b(sd[a-z]{1,2}\d?)\b.*\berror\b'), ] conf['log_name'] = conf.get('log_name', 'drive-audit') logger = get_logger(conf, log_to_console=log_to_console, log_route='drive-audit') devices = get_devices(device_dir, logger) logger.debug("Devices found: %s" % str(devices)) if not devices: logger.error("Error: No devices found!") recon_errors = {} total_errors = 0 for device in devices: recon_errors[device['mount_point']] = 0 errors = get_errors(error_re, log_file_pattern, minutes, logger) logger.debug("Errors found: %s" % str(errors)) unmounts = 0 for kernel_device, count in errors.items(): if count >= error_limit: device = \ [d for d in devices if d['kernel_device'] == kernel_device] if device: mount_point = device[0]['mount_point'] if mount_point.startswith(device_dir): if config_true_value(conf.get('unmount_failed_device', True)): logger.info("Unmounting %s with %d errors" % (mount_point, count)) subprocess.call(['umount', '-fl', mount_point]) logger.info("Commenting out %s from /etc/fstab" % (mount_point)) comment_fstab(mount_point) unmounts += 1 recon_errors[mount_point] = count total_errors += count recon_file = recon_cache_path + "/drive.recon" dump_recon_cache(recon_errors, recon_file, logger) dump_recon_cache({'drive_audit_errors': total_errors}, recon_file, logger) if unmounts == 0: logger.info("No drives were unmounted")