From 8e411618abd588a889dd7634b6ab60709a6680f6 Mon Sep 17 00:00:00 2001 From: f3flight Date: Fri, 5 Aug 2016 16:18:54 +0000 Subject: [PATCH] Fix #52, config changes, new parameter --- doc/source/configuration.rst | 4 +++- rq/default.yaml | 5 +++++ timmy/cli.py | 21 ++++++++++++++------- timmy/conf.py | 9 ++++++--- timmy/env.py | 2 +- timmy/nodes.py | 26 ++++++++++++++++---------- 6 files changed, 45 insertions(+), 22 deletions(-) diff --git a/doc/source/configuration.rst b/doc/source/configuration.rst index 8cae642..e50803e 100644 --- a/doc/source/configuration.rst +++ b/doc/source/configuration.rst @@ -19,7 +19,9 @@ Some of the parameters available in configuration file: * **fuelclient** True/False - whether to use fuelclient library to access Nailgun API * **fuel_skip_proxy** True/False - ignore ``http(s)_proxy`` environment variables when connecting to Nailgun API * **rqdir** the path to the directory containing rqfiles, scripts to execute, and filelists to pass to rsync -* **rqfile** path(s) to rqfile(s) containing actions and/or other configuration parameters. Use list if more than one file is specifed. +* **rqfile** - list of dicts: + * **file** - path to an rqfile containing actions and/or other configuration parameters + * **default** - should always be False, except when included default.yaml is used. This option is used to make **logs_no_default** work * **logs_days** how many past days of logs to collect. This option will set **start** parameter for each **logs** action if not defined in it. * **logs_speed_limit** True/False - enable speed limiting of log transfers (total transfer speed limit, not per-node) * **logs_speed_default** Mbit/s - used when autodetect fails diff --git a/rq/default.yaml b/rq/default.yaml index 51d252d..c0bcb34 100644 --- a/rq/default.yaml +++ b/rq/default.yaml @@ -86,6 +86,11 @@ scripts: centos: [dmesg-centos, yum-list-installed, yum-v-repolist] __default: [ip-ne, iptables, ipnetns, ss, ipa, iptables-nat, df-m, services-status, cpuinfo, df-i, ipro, mount, sysctl-a, pvdisplay, vgdisplay, lvdisplay, lsmod, dmidecode, cat-proc-interrupts, arp-an, uname-a, ps-auxwwf, uptime, dmsetup-info, brctl-show, blkid-o-list] +logs: + __default: + path: '/var/log' + exclude: + - '\.[^12]\.gz$|\.\d{2,}\.gz$' # cmds: # __default: # test: diff --git a/timmy/cli.py b/timmy/cli.py index b6378bc..a528da8 100755 --- a/timmy/cli.py +++ b/timmy/cli.py @@ -114,6 +114,8 @@ def parse_args(): help=('Do not use default log collection parameters,' ' only use what has been provided either via -L' ' or in rqfile(s). Implies "-l".')) + parser.add_argument('--logs-no-fuel-remote', action='store_true', + help='Do not collect remote logs from Fuel.') parser.add_argument('--logs-speed', type=int, metavar='MBIT/S', help=('Limit log collection bandwidth to 90%% of the' ' specified speed in Mbit/s.')) @@ -222,13 +224,16 @@ def main(argv=None): if args.no_clean: conf['clean'] = False if args.rqfile: - conf['rqfile'] = args.rqfile + conf['rqfile'] = [] + for file in args.rqfile: + conf['rqfile'].append({'file': file, 'default': False}) if args.days: - conf['logs'][0]['start'] = args.days conf['logs_days'] = args.days if args.logs_no_default: - conf['logs'] = [] + conf['logs_no_default'] = True args.logs = True + if args.logs_no_fuel_remote: + conf['logs_no_fuel_remote'] = True if args.get_logs: args.logs = True for logs in args.get_logs: @@ -295,11 +300,13 @@ def main(argv=None): nm.calculate_log_size, args=(args.maxthreads,)) if size == 0: logger.warning('Size zero - no logs to collect.') + has_logs = False else: + has_logs = True print('Total logs size to collect: %dMB.' % (size / 1000)) - enough = pretty_run(args.quiet, 'Checking free space', - nm.is_enough_space) - if not enough: + enough_space = pretty_run(args.quiet, 'Checking free space', + nm.is_enough_space) + if not enough_space: logger.error('Not enough space for logs in "%s", exiting.' % nm.conf['archive_dir']) return 2 @@ -315,7 +322,7 @@ def main(argv=None): if not args.no_archive and nm.has(*Node.conf_archive_general): pretty_run(args.quiet, 'Creating outputs and files archive', nm.create_archive_general, args=(60,)) - if (args.only_logs or args.logs) and enough: + if (args.only_logs or args.logs) and has_logs and enough_space: msg = 'Collecting and packing logs' pretty_run(args.quiet, msg, nm.get_logs, args=(conf['compress_timeout'],), diff --git a/timmy/conf.py b/timmy/conf.py index 16251c7..8f35012 100644 --- a/timmy/conf.py +++ b/timmy/conf.py @@ -46,7 +46,8 @@ def load_conf(filename): conf['rqdir'] = os.path.join(dtm, rqdir) else: conf['rqdir'] = rqdir - conf['rqfile'] = os.path.join(conf['rqdir'], rqfile) + conf['rqfile'] = [{'file': os.path.join(conf['rqdir'], rqfile), + 'default': True}] conf['compress_timeout'] = 3600 conf['outdir'] = os.path.join(gettempdir(), 'timmy', 'info') conf['archive_dir'] = os.path.join(gettempdir(), 'timmy', 'archives') @@ -58,8 +59,10 @@ def load_conf(filename): conf['scripts'] = [] conf['files'] = [] conf['filelists'] = [] - conf['logs'] = [{'path': '/var/log', - 'exclude': '\.[^12]\.gz$|\.\d{2,}\.gz$'}] + conf['logs'] = [] + conf['logs_no_default'] = False # skip logs defined in default.yaml + conf['logs_fuel_remote_dir'] = '/var/log/docker-logs/remote' + conf['logs_no_fuel_remote'] = False # do not collect /var/log/remote conf['logs_days'] = 30 conf['logs_speed_limit'] = False # enable speed limiting of log transfers conf['logs_speed_default'] = 100 # Mbit/s, used when autodetect fails diff --git a/timmy/env.py b/timmy/env.py index 887145c..0b62c80 100644 --- a/timmy/env.py +++ b/timmy/env.py @@ -16,7 +16,7 @@ # under the License. project_name = 'timmy' -version = '1.14.5' +version = '1.15.0' if __name__ == '__main__': sys.exit(0) diff --git a/timmy/nodes.py b/timmy/nodes.py index b456d67..995fcf3 100644 --- a/timmy/nodes.py +++ b/timmy/nodes.py @@ -325,11 +325,16 @@ class Node(object): def filter_by_re(item, string): return (('include' not in item or not item['include'] or - re.search(item['include'], string)) and + any([re.search(i, string) for i in item['include']])) and ('exclude' not in item or not item['exclude'] or not - re.search(item['exclude'], string))) + any([re.search(e, string) for e in item['exclude']]))) for item in self.logs: + if self.logs_no_fuel_remote and 'fuel' in self.roles: + self.logger.debug('adding Fuel remote logs to exclude list') + if 'exclude' not in item: + item['exclude'] = [] + item['exclude'].append(self.logs_fuel_remote_dir) start_str = None if 'start' in item or hasattr(self, 'logs_days'): if hasattr(self, 'logs_days') and 'start' not in item: @@ -564,11 +569,15 @@ class NodeManager(object): dst[k][attr] = el[k] def merge_rq(rqfile, dst): - if os.path.sep in rqfile: - src = tools.load_yaml_file(rqfile) + file = rqfile['file'] + if os.path.sep in file: + src = tools.load_yaml_file(file) else: - f = os.path.join(self.rqdir, rqfile) + f = os.path.join(self.rqdir, file) src = tools.load_yaml_file(f) + if self.conf['logs_no_default'] and rqfile['default']: + if 'logs' in src: + src.pop('logs') p = Node.conf_match_prefix once_p = Node.conf_once_prefix + p d = Node.conf_default_key @@ -576,11 +585,8 @@ class NodeManager(object): r_sub(attr, src, attr, d, p, once_p, dst) dst = self.conf - if type(self.conf['rqfile']) is list: - for rqfile in self.conf['rqfile']: - merge_rq(rqfile, dst) - else: - merge_rq(self.conf['rqfile'], dst) + for rqfile in self.conf['rqfile']: + merge_rq(rqfile, dst) def fuel_init(self): if not self.conf['fuel_ip']: