From 6808eace18e64451064dfe5872d1e9f75d59c2b0 Mon Sep 17 00:00:00 2001 From: adobdin Date: Thu, 5 May 2016 13:16:23 +0000 Subject: [PATCH] added: ok_codes for commands replacement: general archive uses gzip fix: flake8 warnings fix: typo from copy/paste functions fix: cmds and files have the same names in dirs fix: tar uses '-C /' option and ok_codes --- rq/cmds/cmds/{dmesg => .dmesg-centos} | 0 rq/cmds/cmds/{dmesg-t => .dmesg-t-ubuntu} | 0 rq/cmds/default/default/.dmesg-centos | 2 +- rq/cmds/default/default/.dmesg-t-ubuntu | 2 +- rq/files/by-os/centos/.keep | 0 rq/files/by-os/centos/yum | 4 +--- rq/files/by-os/ubuntu/.keep | 0 rq/files/by-os/ubuntu/etc-apt | 2 +- rq/files/files/etc-apt | 1 + rq/files/files/yum | 3 +++ timmy/cli.py | 8 ++++---- timmy/conf.py | 8 ++++---- timmy/nodes.py | 23 +++++++++++++---------- 13 files changed, 29 insertions(+), 24 deletions(-) rename rq/cmds/cmds/{dmesg => .dmesg-centos} (100%) rename rq/cmds/cmds/{dmesg-t => .dmesg-t-ubuntu} (100%) delete mode 100644 rq/files/by-os/centos/.keep mode change 100644 => 120000 rq/files/by-os/centos/yum delete mode 100644 rq/files/by-os/ubuntu/.keep mode change 100644 => 120000 rq/files/by-os/ubuntu/etc-apt create mode 100644 rq/files/files/etc-apt create mode 100644 rq/files/files/yum diff --git a/rq/cmds/cmds/dmesg b/rq/cmds/cmds/.dmesg-centos similarity index 100% rename from rq/cmds/cmds/dmesg rename to rq/cmds/cmds/.dmesg-centos diff --git a/rq/cmds/cmds/dmesg-t b/rq/cmds/cmds/.dmesg-t-ubuntu similarity index 100% rename from rq/cmds/cmds/dmesg-t rename to rq/cmds/cmds/.dmesg-t-ubuntu diff --git a/rq/cmds/default/default/.dmesg-centos b/rq/cmds/default/default/.dmesg-centos index 5764359..8ababcf 120000 --- a/rq/cmds/default/default/.dmesg-centos +++ b/rq/cmds/default/default/.dmesg-centos @@ -1 +1 @@ -../../cmds/dmesg \ No newline at end of file +../../cmds/.dmesg-centos \ No newline at end of file diff --git a/rq/cmds/default/default/.dmesg-t-ubuntu b/rq/cmds/default/default/.dmesg-t-ubuntu index c33ba1d..26c060e 120000 --- a/rq/cmds/default/default/.dmesg-t-ubuntu +++ b/rq/cmds/default/default/.dmesg-t-ubuntu @@ -1 +1 @@ -../../cmds/dmesg-t \ No newline at end of file +../../cmds/.dmesg-t-ubuntu \ No newline at end of file diff --git a/rq/files/by-os/centos/.keep b/rq/files/by-os/centos/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/rq/files/by-os/centos/yum b/rq/files/by-os/centos/yum deleted file mode 100644 index 6a3f70d..0000000 --- a/rq/files/by-os/centos/yum +++ /dev/null @@ -1,3 +0,0 @@ -/etc/yum.d/ -/etc/yum -/etc/yum.conf diff --git a/rq/files/by-os/centos/yum b/rq/files/by-os/centos/yum new file mode 120000 index 0000000..9e41c72 --- /dev/null +++ b/rq/files/by-os/centos/yum @@ -0,0 +1 @@ +../../../files/files/yum \ No newline at end of file diff --git a/rq/files/by-os/ubuntu/.keep b/rq/files/by-os/ubuntu/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/rq/files/by-os/ubuntu/etc-apt b/rq/files/by-os/ubuntu/etc-apt deleted file mode 100644 index bc8b814..0000000 --- a/rq/files/by-os/ubuntu/etc-apt +++ /dev/null @@ -1 +0,0 @@ -/etc/apt diff --git a/rq/files/by-os/ubuntu/etc-apt b/rq/files/by-os/ubuntu/etc-apt new file mode 120000 index 0000000..78cdde6 --- /dev/null +++ b/rq/files/by-os/ubuntu/etc-apt @@ -0,0 +1 @@ +../../../files/files/etc-apt \ No newline at end of file diff --git a/rq/files/files/etc-apt b/rq/files/files/etc-apt new file mode 100644 index 0000000..bc8b814 --- /dev/null +++ b/rq/files/files/etc-apt @@ -0,0 +1 @@ +/etc/apt diff --git a/rq/files/files/yum b/rq/files/files/yum new file mode 100644 index 0000000..6a3f70d --- /dev/null +++ b/rq/files/files/yum @@ -0,0 +1,3 @@ +/etc/yum.d/ +/etc/yum +/etc/yum.conf diff --git a/timmy/cli.py b/timmy/cli.py index 5dbb21a..0e05e51 100755 --- a/timmy/cli.py +++ b/timmy/cli.py @@ -74,13 +74,13 @@ def main(argv=None): config = Conf() if args.config: config = Conf.load_conf(args.config) - main_arc = os.path.join(config.archives, 'general.tar.bz2') + main_arc = os.path.join(config.archives, 'general.tar.gz') if args.dest_file: main_arc = args.dest_file nm = NodeManager(conf=config, - extended=args.extended, - cluster=args.cluster, - ) + extended=args.extended, + cluster=args.cluster, + ) if not args.only_logs: nm.get_node_file_list() nm.launch_ssh(config.outdir, args.maxthreads) diff --git a/timmy/conf.py b/timmy/conf.py index 2813839..2b1198a 100644 --- a/timmy/conf.py +++ b/timmy/conf.py @@ -1,8 +1,7 @@ -import logging -import sys from nodefilter import NodeFilter from tools import load_yaml_file + class Conf(object): """Configuration parameters""" hard_filter = None @@ -20,8 +19,8 @@ class Conf(object): compress_timeout = 3600 archives = '/tmp/timmy/archives' cmds_archive = '' - logs = { 'path': '/var/log', - 'exclude': '[-_]\d{8}$|atop[-_]|\.gz$'} + logs = {'path': '/var/log', + 'exclude': '[-_]\d{8}$|atop[-_]|\.gz$'} def __init__(self, **entries): self.__dict__.update(entries) @@ -37,5 +36,6 @@ class Conf(object): if __name__ == '__main__': + import yaml conf = Conf.load_conf('config.yaml') print(yaml.dump(conf)) diff --git a/timmy/nodes.py b/timmy/nodes.py index e143f00..65f02c9 100644 --- a/timmy/nodes.py +++ b/timmy/nodes.py @@ -108,7 +108,6 @@ class Node(object): def set_files_from_yaml(self, dirname, key, ds, version): files = [] dfs = 'default' - print(ds) for role in self.roles: if 'by-role' in ds[key] and role in ds[key]['by-role']: for f in ds[key]['by-role'][role]: @@ -187,7 +186,8 @@ class Node(object): logging.error("exec_cmd: can't write to file %s" % dfile) return self - def exec_simple_cmd(self, cmd, infile, outfile, timeout=15, fake=False): + def exec_simple_cmd(self, cmd, infile, outfile, timeout=15, + fake=False, ok_codes=[0, ]): logging.info('node:%s(%s), exec: %s' % (self.node_id, self.ip, cmd)) if not fake: outs, errs, code = tools.ssh_node(ip=self.ip, @@ -197,7 +197,7 @@ class Node(object): timeout=timeout, outputfile=outfile, inputfile=infile) - if code != 0: + if code not in ok_codes: logging.warning("node: %s, ip: %s, cmdfile: %s," " code: %s, error message: %s" % (self.node_id, self.ip, cmd, code, errs)) @@ -264,7 +264,7 @@ class Node(object): if '\t' in line: size, f = line.split('\t') if filter_by_re(item, f): - item['files'][filename] = int(size) + item['files'][f] = int(size) logging.debug('logs_populate: logs: %s' % (item['files'])) return self @@ -305,7 +305,6 @@ class NodeManager(object): if (not os.path.exists(self.dirname)): logging.error("directory %s doesn't exist" % (self.dirname)) sys.exit(1) - dn = os.path.basename(self.dirname) self.files = tools.load_yaml_file(conf.rqfile) if (conf.fuelip is None) or (conf.fuelip == ""): logging.error('looks like fuelip is not set(%s)' % conf.fuelip) @@ -462,7 +461,10 @@ class NodeManager(object): # ### case roles = [] for node in self.nodes.values(): - node.set_files_from_yaml(self.dirname, key, self.files, self.version) + node.set_files_from_yaml(self.dirname, + key, + self.files, + self.version) # once-by-role functionality if self.extended and key == ckey and node.online: for role in node.roles: @@ -550,7 +552,7 @@ class NodeManager(object): return True def create_archive_general(self, directory, outfile, timeout): - cmd = "tar jcf '%s' -C %s %s" % (outfile, directory, ".") + cmd = "tar zcf '%s' -C %s %s" % (outfile, directory, ".") tools.mdir(self.conf.archives) logging.debug("create_archive_general: cmd: %s" % cmd) outs, errs, code = tools.launch_cmd(command=cmd, @@ -598,19 +600,20 @@ class NodeManager(object): try: with open(logslistfile, 'w') as llf: for filename in node.logs_dict(): - llf.write(filename+"\0") + llf.write(filename.lstrip('/')+"\0") except: logging.error("create_archive_logs: Can't write to file %s" % logslistfile) continue - cmd = ("tar --gzip --create --warning=no-file-changed " + cmd = ("tar --gzip -C / --create --warning=no-file-changed " " --file - --null --files-from -") if not (node.ip == 'localhost' or node.ip.startswith('127.')): cmd = ' '.join([cmd, "| python -c '%s'" % pythonslowpipe]) args = {'cmd': cmd, 'infile': logslistfile, 'outfile': node.archivelogsfile, - 'timeout': timeout} + 'timeout': timeout, + 'ok_codes': [0, 1]} run_items.append(tools.RunItem(target=node.exec_simple_cmd, args=args)) tools.run_batch(run_items, maxthreads)