Improve output, fix warnings, incr. def. loglevel, clean by default

rq.yaml

@@ -4,10 +4,12 @@ filelists:
     ceph-osd: [etc-ceph]
     cinder: [etc-cinder]
     compute: [etc-nova, etc-libvirt]
-    controller: [etc-glance, etc-haproxy, etc-heat, etc-ceph-controller, etc-nova,
+    controller: [etc-glance, etc-haproxy, etc-nova,
       etc-keystone, etc-neutron, etc-mysql]
+    # f3flight: removed etc-heat, etc-ceph-controller from controller since it's not always present, gives a warning
+    # f3flight: need to make a better way to decide which files to collect
   by_os_platform:
-    centos: [yum]
+    centos: [etc-yum]
     ubuntu: [etc-apt]
 scripts:
   by_release:

rq/filelists/etc-yum

@@ -0,0 +1 @@
+/etc/yum

rq/filelists/etc-yum-d

@@ -0,0 +1 @@
+/etc/yum.d

rq/filelists/yum

@@ -1,3 +0,0 @@
-/etc/yum.d/
-/etc/yum
-/etc/yum.conf

rq/scripts/ceph-mon-status

@@ -1,2 +1,7 @@
-ceph mon_status
+which ceph
-
+if [ "$?" -eq "0" ]
+then
+  ceph mon_status
+else
+  echo 'Timmy says: ceph not installed'
+fi

timmy/cli.py

@@ -26,6 +26,14 @@ from timmy.tools import interrupt_wrapper
 from tempfile import gettempdir
 
 
+def pretty_run(msg, f, args=[], kwargs={}):
+    sys.stdout.write('%s...\r' % msg)
+    sys.stdout.flush()
+    result = f(*args, **kwargs)
+    print('%s: done' % msg)
+    return result
+
+
 @interrupt_wrapper
 def main(argv=None):
     if argv is None:
@@ -82,8 +90,12 @@ def main(argv=None):
                               ' an archive with all outputs and files'
                               ' is created every time you run Timmy.'),
                         action='store_true')
+    parser.add_argument('--no-clean',
+                        help=('Do not clean previous results. Allows'
+                              ' accumulating results across runs.'),
+                        action='store_true')
     args = parser.parse_args(argv[1:])
-    loglevel = logging.ERROR
+    loglevel = logging.WARNING
     if args.verbose:
         loglevel = logging.INFO
     if args.debug:
@@ -94,10 +106,9 @@ def main(argv=None):
     conf = load_conf(args.conf)
     if args.command or args.file:
         conf['shell_mode'] = True
+    if args.no_clean:
+        conf['clean'] = False
     if conf['shell_mode']:
-        # set clean to True if not defined in config
-        if conf['clean'] is None:
-            conf['clean'] = True
         filter = conf['hard_filter']
         # config cleanup for shell mode
         for k in Node.conf_actionable:
@@ -118,28 +129,44 @@ def main(argv=None):
     main_arc = os.path.join(conf['archives'], 'general.tar.gz')
     if args.dest_file:
         main_arc = args.dest_file
-    nm = NodeManager(conf=conf,
+    nm = pretty_run('Initializing node data',
-                     extended=args.extended)
+                    NodeManager,
+                    kwargs={'conf': conf,
+                     'extended': args.extended})
     if not args.only_logs:
-        nm.run_commands(conf['outdir'], args.maxthreads)
+        if not (conf['shell_mode'] and not args.command):
-        nm.get_files(conf['outdir'], args.maxthreads)
+            pretty_run('Executing commands and scripts',
+                        nm.run_commands,
+                        args=(conf['outdir'], args.maxthreads))
+        if not (conf['shell_mode'] and not args.file):
+            pretty_run('Collecting files and filelists',
+                        nm.get_files,
+                        args=(conf['outdir'], args.maxthreads))
         if not args.no_archive:
-            nm.create_archive_general(conf['outdir'],
+            pretty_run('Creating outputs and files archive',
-                                      main_arc,
+                        nm.create_archive_general,
-                                      60)
+                        args=(conf['outdir'], main_arc, 60))
     if args.only_logs or args.getlogs:
         lf = os.path.join(gettempdir(), 'timmy-logs.lock')
         lock = flock.FLock(lf)
         if lock.lock():
-            size = nm.calculate_log_size(args.maxthreads)
+            size = pretty_run('Calculating logs size',
+                              nm.calculate_log_size,
+                              args=(args.maxthreads,))
             if size == 0:
-                logging.warning('No logs to collect.')
+                logging.warning('Size zero - no logs to collect.')
+                print('Size zero - no logs to collect.')
                 return
-            if nm.is_enough_space(conf['archives']):
+            enough = pretty_run('Checking free space',
-                nm.get_logs(conf['archives'],
+                                 nm.is_enough_space,
-                            conf['compress_timeout'],
+                                 args=(conf['archives'],))
-                            maxthreads=args.logs_maxthreads,
+            if enough:
-                            fake=args.fake_logs)
+                pretty_run('Collecting and packing logs',
+                           nm.get_logs,
+                           args=(conf['archives'],
+                                 conf['compress_timeout']),
+                           kwargs={'maxthreads': args.logs_maxthreads,
+                                   'fake': args.fake_logs})
             lock.unlock()
         else:
             logging.warning('Unable to obtain lock %s, skipping "logs"-part' %

timmy/conf.py

@@ -41,9 +41,8 @@ def load_conf(filename):
     place specified by conf['outdir'], archive will also be created and put
     in a place specified by conf['archives'].'''
     conf['shell_mode'] = False
-    '''Clean - erase previous results in outdir and archives dir, if any.
+    '''Clean - erase previous results in outdir and archives dir, if any.'''
-    Enabled by default for shell mode. Set to True or False to override.'''
+    conf['clean'] = True
-    conf['clean'] = None
     if filename:
         conf_extra = load_yaml_file(filename)
         conf.update(**conf_extra)

timmy/nodes.py

@@ -214,7 +214,7 @@ class Node(object):
                                 (self.id, self.ip, cmd, code, errs))
 
     def get_files(self, odir='info', timeout=15):
-        def check_code(code):
+        def check_code(code, errs):
             if code != 0:
                 logging.warning("get_files: node: %s, ip: %s, "
                                 "code: %s, error message: %s" %
@@ -232,7 +232,7 @@ class Node(object):
                                                       file=file,
                                                       ddir=ddir,
                                                       recursive=True)
-                check_code(code)
+                check_code(code, errs)
         else:
             data = ''
             for f in self.filelists:
@@ -252,7 +252,7 @@ class Node(object):
                                                 ssh_opts=self.ssh_opts,
                                                 dpath=ddir,
                                                 timeout=self.timeout)
-            check_code(code)
+            check_code(c, e)
 
     def logs_populate(self, timeout=5):
 

timmy/tools.py

@@ -192,12 +192,14 @@ def launch_cmd(command, timeout, input=None):
         timeout_killer = threading.Timer(timeout, _timeout_terminate, [p.pid])
         timeout_killer.start()
         outs, errs = p.communicate(input=input)
+        errs = errs.rstrip('\n')
     except:
         try:
             p.kill()
         except:
             pass
         outs, errs = p.communicate()
+        errs = errs.rstrip('\n')
         logging.error("command: %s err: %s, returned: %s" %
                       (command, errs, p.returncode))
     finally: