replace directories structure by yaml config file

fix: typo in cli.py

config.yaml

@@ -10,6 +10,7 @@ env_vars:
     - 'IPTABLES_STR="iptables -nvL"'
 fuelip:  '127.0.0.1'
 rqdir: './rq'
+rqfile: './rq.yaml'
 soft_filter:
     status:  ['ready']
 timeout: 15

rq.yaml

@@ -0,0 +1,53 @@
+files:
+  by-role:
+    fuel: [etc-nailgun, etc-fuel]
+    ceph-osd: [etc-ceph]
+    cinder: [etc-cinder]
+    compute: [etc-nova, etc-libvirt]
+    controller: [etc-glance, etc-haproxy, etc-heat, etc-ceph-controller, etc-nova,
+      etc-keystone, etc-neutron, etc-mysql]
+  by-os:
+    centos: [yum]
+    ubuntu: [etc-apt]
+  files: [etc-nailgun, etc-glance, etc-haproxy, etc-fuel, etc-heat, etc-ceph-controller,
+    etc-nova, etc-keystone, etc-ceph, etc-neutron, etc-mysql, etc-libvirt, etc-cinder]
+cmds:
+  release-6.1:
+    fuel: [fuel-notifications]
+  release-7.0:
+    fuel: [fuel-notifications]
+  release-5.1.1:
+    fuel: [fuel-dockerctl-list, fuel-docker-ps, fuel-dockerctl-check, fuel-docker-db-archive]
+  by-role:
+    fuel: [fuel-release, fuel-task-list, fuel-environment-list]
+    cinder: [ovs-vsctl-show, cinder-manage]
+    compute: [compute-iptables-nat, ovs-dump-flows, compute-iptables, ovs-ofctl-show-bridges,
+      ovs-vsctl-show]
+    controller: [rabbitmqctl-list-queues, nova-manage-service-list, iptables-namespaces,
+      rabbitmqctl-cluster-status, crm-resource-status, ovs-dump-flows, neutron-agent-list,
+      mysql-status, ceph_mon_status, ovs-ofctl-show-bridges, rabbitmqctl_list_connections,
+      ovs-vsctl-show, rabbitmqctl-report, mysql-size, rabbitmqctl-status, crm-resource-list,
+      cinder-manage]
+    mongo: [mongo-replication-status, ipa, mongo-replica-conf, mongo-status, ovs-vsctl-show]
+  once-by-role:
+    ceph-osd: [ceph-df, ceph-osd-status, ceph-osd-tree, ceph-pg-dump, ovs-vsctl-show,
+      ceph-health-detail]
+    controller: [neutron-router-list, neutron-net-list, neutron-subnet-list, keystone-endpoint-list,
+      cinder-list, nova-list, nova-manage-vm-list, keystone-tenant-list, nova-usage-list,
+      neutron-port-list]
+  release-4.1.1:
+    fuel: [fuel-postgres-dump]
+  release-8.0:
+    fuel: [fuel-notifications]
+  release-5.0.1:
+    fuel: [fuel-docker-ps, fuel-dockerctl-check, fuel-docker-db-archive]
+  default:
+    [.packages-ubuntu, ip-ne, iptables, ipnetns, ss, ipa, iptables-nat,
+    .dmesg-t-ubuntu, df-m, services-status, .dmesg-centos, cpuinfo, df-i, .packages-centos,
+    ipro]
+  release-6.0:
+    compute: [ipset-save, ipset-list]
+    controller: [ipset-save, ipset-list]
+#  by-os:
+#    centos: []
+#    ubuntu: []

timmy/cli.py

@@ -77,27 +77,27 @@ def main(argv=None):
     main_arc = os.path.join(config.archives, 'general.tar.bz2')
     if args.dest_file:
         main_arc = args.dest_file
-    n = NodeManager(conf=config,
+    nm = NodeManager(conf=config,
                     extended=args.extended,
                     cluster=args.cluster,
                     )
     if not args.only_logs:
-        n.get_node_file_list()
+        nm.get_node_file_list()
-        n.launch_ssh(config.outdir, args.maxthreads)
+        nm.launch_ssh(config.outdir, args.maxthreads)
-        n.get_conf_files(config.outdir, args.maxthreads)
+        nm.get_conf_files(config.outdir, args.maxthreads)
-        n.create_archive_general(config.outdir,
+        nm.create_archive_general(config.outdir,
                                   main_arc,
                                   60)
     if args.only_logs or args.getlogs:
         lf = '/tmp/timmy-logs.lock'
         lock = flock.FLock(lf)
         if lock.lock():
-            size = n.calculate_log_size(args.maxthreads)
+            size = nm.calculate_log_size(args.maxthreads)
             if size == 0:
                 logging.warning('No logs to collect.')
                 return
-            if n.is_enough_space(config.archives):
+            if nm.is_enough_space(config.archives):
-                n.archive_logs(config.archives,
+                nm.archive_logs(config.archives,
                                 config.compress_timeout,
                                 maxthreads=args.logs_maxthreads,
                                 fake=args.fake_logs)
@@ -105,8 +105,8 @@ def main(argv=None):
         else:
             logging.warning('Unable to obtain lock %s, skipping "logs"-part' %
                             lf)
-    logging.info("Nodes:\n%s" % n)
+    logging.info("Nodes:\n%s" % nm)
-    print(n)
+    print(nm)
     return 0
 
 if __name__ == '__main__':

timmy/conf.py

@@ -16,6 +16,7 @@ class Conf(object):
     outdir = '/tmp/timmy/info'
     timeout = 15
     rqdir = '/usr/share/timmy/rq'
+    rqfile = '/usr/share/timmy/rq.yaml'
     compress_timeout = 3600
     archives = '/tmp/timmy/archives'
     cmds_archive = ''

timmy/nodes.py

@@ -105,6 +105,28 @@ class Node(object):
         logging.debug('set_files:\nkey: %s, node: %s, file_list: %s' %
                       (key, self.node_id, self.files[key]))
 
+    def set_files_from_yaml(self, dirname, key, ds, version):
+        files = []
+        dfs = 'default'
+        print(ds)
+        for role in self.roles:
+            if 'by-role' in ds[key] and role in ds[key]['by-role']:
+                for f in ds[key]['by-role'][role]:
+                    files += [os.path.join(dirname, key, key, f)]
+            if (('release-'+version in ds[key]) and
+                    (role in ds[key]['release-'+version])):
+                for f in ds[key]['release-'+version][role]:
+                        files += [os.path.join(dirname, key, key, f)]
+            if 'by-os' in ds[key]:
+                for f in ds[key]['by-os'][self.os_platform]:
+                    files += [os.path.join(dirname, key, key, f)]
+            if dfs in ds[key]:
+                for f in ds[key][dfs]:
+                    files += [os.path.join(dirname, key, key, f)]
+        self.files[key] = sorted(set(files))
+        logging.debug('set_files:\nkey: %s, node: %s, file_list: %s' %
+                      (key, self.node_id, self.files[key]))
+
     def checkos(self, filename):
         bname = str(os.path.basename(filename))
         logging.debug('check os: node: %s, filename %s' %
@@ -284,7 +306,7 @@ class NodeManager(object):
             logging.error("directory %s doesn't exist" % (self.dirname))
             sys.exit(1)
         dn = os.path.basename(self.dirname)
-        self.files = tools.get_dir_structure(conf.rqdir)[dn]
+        self.files = tools.load_yaml_file(conf.rqfile)
         if (conf.fuelip is None) or (conf.fuelip == ""):
             logging.error('looks like fuelip is not set(%s)' % conf.fuelip)
             sys.exit(7)
@@ -440,7 +462,7 @@ class NodeManager(object):
             #  ###   case
             roles = []
             for node in self.nodes.values():
-                node.set_files(self.dirname, key, self.files, self.version)
+                node.set_files_from_yaml(self.dirname, key, self.files, self.version)
                 # once-by-role functionality
                 if self.extended and key == ckey and node.online:
                     for role in node.roles: