x/timmy
Code Issues Proposed changes

added: ok_codes for commands

Browse Source 
replacement: general archive uses gzip

fix: flake8 warnings
fix: typo from copy/paste functions
fix: cmds and files have the same names in dirs
fix: tar uses '-C /' option and ok_codes
This commit is contained in:
adobdin 2016-05-05 13:16:23 +00:00
parent 1a76e7a6b7
commit 6808eace18
13 changed files with 29 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
rq/cmds/cmds/dmesg → rq/cmds/cmds/.dmesg-centos
View File

0
rq/cmds/cmds/dmesg-t → rq/cmds/cmds/.dmesg-t-ubuntu
View File

2
rq/cmds/default/default/.dmesg-centos
View File

@ -1 +1 @@
../../cmds/dmesg
../../cmds/.dmesg-centos

2
rq/cmds/default/default/.dmesg-t-ubuntu
View File

@ -1 +1 @@
../../cmds/dmesg-t
../../cmds/.dmesg-t-ubuntu

0
rq/files/by-os/centos/.keep
View File

3
rq/files/by-os/centos/yum
View File

@ -1,3 +0,0 @@
/etc/yum.d/
/etc/yum
/etc/yum.conf

1
rq/files/by-os/centos/yum Symbolic link
View File

@ -0,0 +1 @@
../../../files/files/yum

0
rq/files/by-os/ubuntu/.keep
View File

1
rq/files/by-os/ubuntu/etc-apt
View File

@ -1 +0,0 @@
/etc/apt

1
rq/files/by-os/ubuntu/etc-apt Symbolic link
View File

@ -0,0 +1 @@
../../../files/files/etc-apt

1
rq/files/files/etc-apt Normal file
View File

@ -0,0 +1 @@
/etc/apt

3
rq/files/files/yum Normal file
View File

@ -0,0 +1,3 @@
/etc/yum.d/
/etc/yum
/etc/yum.conf

8
timmy/cli.py
View File

@ -74,13 +74,13 @@ def main(argv=None):
config = Conf()
if args.config:
config = Conf.load_conf(args.config)
main_arc = os.path.join(config.archives, 'general.tar.bz2')
main_arc = os.path.join(config.archives, 'general.tar.gz')
if args.dest_file:
main_arc = args.dest_file
nm = NodeManager(conf=config,
extended=args.extended,
cluster=args.cluster,
)
extended=args.extended,
cluster=args.cluster,
)
if not args.only_logs:
nm.get_node_file_list()
nm.launch_ssh(config.outdir, args.maxthreads)

8
timmy/conf.py
View File

@ -1,8 +1,7 @@
import logging
import sys
from nodefilter import NodeFilter
from tools import load_yaml_file
class Conf(object):
"""Configuration parameters"""
hard_filter = None
@ -20,8 +19,8 @@ class Conf(object):
compress_timeout = 3600
archives = '/tmp/timmy/archives'
cmds_archive = ''
logs = { 'path': '/var/log',
'exclude': '[-_]\d{8}$|atop[-_]|\.gz$'}
logs = {'path': '/var/log',
'exclude': '[-_]\d{8}$|atop[-_]|\.gz$'}
def __init__(self, **entries):
self.__dict__.update(entries)
@ -37,5 +36,6 @@ class Conf(object):
if __name__ == '__main__':
import yaml
conf = Conf.load_conf('config.yaml')
print(yaml.dump(conf))

23
timmy/nodes.py
View File

@ -108,7 +108,6 @@ class Node(object):
def set_files_from_yaml(self, dirname, key, ds, version):
files = []
dfs = 'default'
print(ds)
for role in self.roles:
if 'by-role' in ds[key] and role in ds[key]['by-role']:
for f in ds[key]['by-role'][role]:
@ -187,7 +186,8 @@ class Node(object):
logging.error("exec_cmd: can't write to file %s" % dfile)
return self
def exec_simple_cmd(self, cmd, infile, outfile, timeout=15, fake=False):
def exec_simple_cmd(self, cmd, infile, outfile, timeout=15,
fake=False, ok_codes=[0, ]):
logging.info('node:%s(%s), exec: %s' % (self.node_id, self.ip, cmd))
if not fake:
outs, errs, code = tools.ssh_node(ip=self.ip,
@ -197,7 +197,7 @@ class Node(object):
timeout=timeout,
outputfile=outfile,
inputfile=infile)
if code != 0:
if code not in ok_codes:
logging.warning("node: %s, ip: %s, cmdfile: %s,"
" code: %s, error message: %s" %
(self.node_id, self.ip, cmd, code, errs))
@ -264,7 +264,7 @@ class Node(object):
if '\t' in line:
size, f = line.split('\t')
if filter_by_re(item, f):
item['files'][filename] = int(size)
item['files'][f] = int(size)
logging.debug('logs_populate: logs: %s' % (item['files']))
return self
@ -305,7 +305,6 @@ class NodeManager(object):
if (not os.path.exists(self.dirname)):
logging.error("directory %s doesn't exist" % (self.dirname))
sys.exit(1)
dn = os.path.basename(self.dirname)
self.files = tools.load_yaml_file(conf.rqfile)
if (conf.fuelip is None) or (conf.fuelip == ""):
logging.error('looks like fuelip is not set(%s)' % conf.fuelip)
@ -462,7 +461,10 @@ class NodeManager(object):
# ### case
roles = []
for node in self.nodes.values():
node.set_files_from_yaml(self.dirname, key, self.files, self.version)
node.set_files_from_yaml(self.dirname,
key,
self.files,
self.version)
# once-by-role functionality
if self.extended and key == ckey and node.online:
for role in node.roles:
@ -550,7 +552,7 @@ class NodeManager(object):
return True
def create_archive_general(self, directory, outfile, timeout):
cmd = "tar jcf '%s' -C %s %s" % (outfile, directory, ".")
cmd = "tar zcf '%s' -C %s %s" % (outfile, directory, ".")
tools.mdir(self.conf.archives)
logging.debug("create_archive_general: cmd: %s" % cmd)
outs, errs, code = tools.launch_cmd(command=cmd,
@ -598,19 +600,20 @@ class NodeManager(object):
try:
with open(logslistfile, 'w') as llf:
for filename in node.logs_dict():
llf.write(filename+"\0")
llf.write(filename.lstrip('/')+"\0")
except:
logging.error("create_archive_logs: Can't write to file %s" %
logslistfile)
continue
cmd = ("tar --gzip --create --warning=no-file-changed "
cmd = ("tar --gzip -C / --create --warning=no-file-changed "
" --file - --null --files-from -")
if not (node.ip == 'localhost' or node.ip.startswith('127.')):
cmd = ' '.join([cmd, "| python -c '%s'" % pythonslowpipe])
args = {'cmd': cmd,
'infile': logslistfile,
'outfile': node.archivelogsfile,
'timeout': timeout}
'timeout': timeout,
'ok_codes': [0, 1]}
run_items.append(tools.RunItem(target=node.exec_simple_cmd,
args=args))
tools.run_batch(run_items, maxthreads)