Fix #52, config changes, new parameter

doc/source/configuration.rst

@@ -19,7 +19,9 @@ Some of the parameters available in configuration file:
 * **fuelclient** True/False - whether to use fuelclient library to access Nailgun API
 * **fuel_skip_proxy** True/False - ignore ``http(s)_proxy`` environment variables when connecting to Nailgun API
 * **rqdir** the path to the directory containing rqfiles, scripts to execute, and filelists to pass to rsync
-* **rqfile** path(s) to rqfile(s) containing actions and/or other configuration parameters. Use list if more than one file is specifed.
+* **rqfile** - list of dicts:
+    * **file** - path to an rqfile containing actions and/or other configuration parameters
+    * **default** - should always be False, except when included default.yaml is used. This option is used to make **logs_no_default** work
 * **logs_days** how many past days of logs to collect. This option will set **start** parameter for each **logs** action if not defined in it.
 * **logs_speed_limit** True/False - enable speed limiting of log transfers (total transfer speed limit, not per-node)
 * **logs_speed_default** Mbit/s - used when autodetect fails

rq/default.yaml

@@ -86,6 +86,11 @@ scripts:
       centos: [dmesg-centos, yum-list-installed, yum-v-repolist]
   __default:
     [ip-ne, iptables, ipnetns, ss, ipa, iptables-nat, df-m, services-status, cpuinfo, df-i, ipro, mount, sysctl-a, pvdisplay, vgdisplay, lvdisplay, lsmod, dmidecode, cat-proc-interrupts, arp-an, uname-a, ps-auxwwf, uptime, dmsetup-info, brctl-show, blkid-o-list]
+logs:
+  __default:
+    path: '/var/log'
+    exclude:
+      - '\.[^12]\.gz$|\.\d{2,}\.gz$'
 # cmds:
 #   __default:
 #     test:

timmy/cli.py

@@ -114,6 +114,8 @@ def parse_args():
                         help=('Do not use default log collection parameters,'
                               ' only use what has been provided either via -L'
                               ' or in rqfile(s). Implies "-l".'))
+    parser.add_argument('--logs-no-fuel-remote', action='store_true',
+                        help='Do not collect remote logs from Fuel.')
     parser.add_argument('--logs-speed', type=int, metavar='MBIT/S',
                         help=('Limit log collection bandwidth to 90%% of the'
                               ' specified speed in Mbit/s.'))
@@ -222,13 +224,16 @@ def main(argv=None):
     if args.no_clean:
         conf['clean'] = False
     if args.rqfile:
-        conf['rqfile'] = args.rqfile
+        conf['rqfile'] = []
+        for file in args.rqfile:
+            conf['rqfile'].append({'file': file, 'default': False})
     if args.days:
-        conf['logs'][0]['start'] = args.days
         conf['logs_days'] = args.days
     if args.logs_no_default:
-        conf['logs'] = []
+        conf['logs_no_default'] = True
         args.logs = True
+    if args.logs_no_fuel_remote:
+        conf['logs_no_fuel_remote'] = True
     if args.get_logs:
         args.logs = True
         for logs in args.get_logs:
@@ -295,11 +300,13 @@ def main(argv=None):
                           nm.calculate_log_size, args=(args.maxthreads,))
         if size == 0:
             logger.warning('Size zero - no logs to collect.')
+            has_logs = False
         else:
+            has_logs = True
             print('Total logs size to collect: %dMB.' % (size / 1000))
-            enough = pretty_run(args.quiet, 'Checking free space',
+            enough_space = pretty_run(args.quiet, 'Checking free space',
-                                nm.is_enough_space)
+                                      nm.is_enough_space)
-            if not enough:
+            if not enough_space:
                 logger.error('Not enough space for logs in "%s", exiting.' %
                              nm.conf['archive_dir'])
                 return 2
@@ -315,7 +322,7 @@ def main(argv=None):
         if not args.no_archive and nm.has(*Node.conf_archive_general):
             pretty_run(args.quiet, 'Creating outputs and files archive',
                        nm.create_archive_general, args=(60,))
-    if (args.only_logs or args.logs) and enough:
+    if (args.only_logs or args.logs) and has_logs and enough_space:
         msg = 'Collecting and packing logs'
         pretty_run(args.quiet, msg, nm.get_logs,
                    args=(conf['compress_timeout'],),

timmy/conf.py

@@ -46,7 +46,8 @@ def load_conf(filename):
         conf['rqdir'] = os.path.join(dtm, rqdir)
     else:
         conf['rqdir'] = rqdir
-    conf['rqfile'] = os.path.join(conf['rqdir'], rqfile)
+    conf['rqfile'] = [{'file': os.path.join(conf['rqdir'], rqfile),
+                      'default': True}]
     conf['compress_timeout'] = 3600
     conf['outdir'] = os.path.join(gettempdir(), 'timmy', 'info')
     conf['archive_dir'] = os.path.join(gettempdir(), 'timmy', 'archives')
@@ -58,8 +59,10 @@ def load_conf(filename):
     conf['scripts'] = []
     conf['files'] = []
     conf['filelists'] = []
-    conf['logs'] = [{'path': '/var/log',
+    conf['logs'] = []
-                    'exclude': '\.[^12]\.gz$|\.\d{2,}\.gz$'}]
+    conf['logs_no_default'] = False  # skip logs defined in default.yaml
+    conf['logs_fuel_remote_dir'] = '/var/log/docker-logs/remote'
+    conf['logs_no_fuel_remote'] = False  # do not collect /var/log/remote
     conf['logs_days'] = 30
     conf['logs_speed_limit'] = False  # enable speed limiting of log transfers
     conf['logs_speed_default'] = 100  # Mbit/s, used when autodetect fails

timmy/env.py

@@ -16,7 +16,7 @@
 #    under the License.
 
 project_name = 'timmy'
-version = '1.14.5'
+version = '1.15.0'
 
 if __name__ == '__main__':
     sys.exit(0)

timmy/nodes.py

@@ -325,11 +325,16 @@ class Node(object):
 
         def filter_by_re(item, string):
             return (('include' not in item or not item['include'] or
-                     re.search(item['include'], string)) and
+                     any([re.search(i, string) for i in item['include']])) and
                     ('exclude' not in item or not item['exclude'] or not
-                     re.search(item['exclude'], string)))
+                     any([re.search(e, string) for e in item['exclude']])))
 
         for item in self.logs:
+            if self.logs_no_fuel_remote and 'fuel' in self.roles:
+                self.logger.debug('adding Fuel remote logs to exclude list')
+                if 'exclude' not in item:
+                    item['exclude'] = []
+                item['exclude'].append(self.logs_fuel_remote_dir)
             start_str = None
             if 'start' in item or hasattr(self, 'logs_days'):
                 if hasattr(self, 'logs_days') and 'start' not in item:
@@ -564,11 +569,15 @@ class NodeManager(object):
                 dst[k][attr] = el[k]
 
         def merge_rq(rqfile, dst):
-            if os.path.sep in rqfile:
+            file = rqfile['file']
-                src = tools.load_yaml_file(rqfile)
+            if os.path.sep in file:
+                src = tools.load_yaml_file(file)
             else:
-                f = os.path.join(self.rqdir, rqfile)
+                f = os.path.join(self.rqdir, file)
                 src = tools.load_yaml_file(f)
+            if self.conf['logs_no_default'] and rqfile['default']:
+                if 'logs' in src:
+                    src.pop('logs')
             p = Node.conf_match_prefix
             once_p = Node.conf_once_prefix + p
             d = Node.conf_default_key
@@ -576,11 +585,8 @@ class NodeManager(object):
                 r_sub(attr, src, attr, d, p, once_p, dst)
 
         dst = self.conf
-        if type(self.conf['rqfile']) is list:
+        for rqfile in self.conf['rqfile']:
-            for rqfile in self.conf['rqfile']:
+            merge_rq(rqfile, dst)
-                merge_rq(rqfile, dst)
-        else:
-            merge_rq(self.conf['rqfile'], dst)
 
     def fuel_init(self):
         if not self.conf['fuel_ip']: