Refactor read config file

In order to make configuration more clearly,
seperating "network_types" configuration into serveral configuration.

Add timeout decorator to make sure program can goes more smoothly.

Change-Id: I0d9765b23f89e2c8ac8b1d9047cfebb89f04df8d

etc/steth.conf

@@ -1,26 +1,34 @@
 [DEFAULT]
-# (ListOpt) list of networks types.
+#(StrOpt) Name prefix of every node. By default, this value
-# We may have multi network types in one node, such as mgmt, net and stroage.
-# so this value should be a list.
-# We seperate each item by ":". Treat first item as network type.
-# The second is physical nic name. And the third is network_prefix.
-# Example: "mgmt:eth0:1.1.1.,net:eth1:2.2.2.,storage:eth2:3.3.3."
-network_types=mgmt:eth0:1.1.1.,net:eth1:2.2.2.,storage:eth2:3.3.3.
-
-# (ListOpt) All nodes info. Just need sequence number.
-# Example: 64, 65, 66
-nodes_id=39,233,64,65,66
-
-# (StrOpt) Name prefix of every node. By default, this value
 # is "server". We combine "node_name_prefix" with
 # "nodes_id", to define nodes. Such as "server-64", "server-68"
 # and so on. In every region, we give every node a specific name.
 # Ensure that DNS can resolve the nodes.
 node_name_prefix=server-
 
-# (StrOpt) Name of physical interface in each network node or
+[mgmt_type_network]
-# compute node.
+# (StrOpt) String of physical ethernet name.
-physical_interface=eth3
+mgmt_ethernet_name = eth0
+# (StrOpt) String of <ip_addr_start>:<ip_addr_end>
+# tuples specifying the managment network.
+# mgmt_network_ranges = eth0:1.1.1.1:1.1.1.10
+mgmt_network_ranges = 1.1.1.1:1.1.1.2
+
+[sdn_type_network]
+# (StrOpt) String of physical ethernet name.
+sdn_ethernet_name = eth1
+# (StrOpt) String of <ip_addr_start>:<ip_addr_end>
+# tuples specifying the sdn network.
+# sdn_network_ranges = 2.2.2.2:2.2.2.10
+sdn_network_ranges = 2.2.2.1:2.2.2.2
+
+[storage_type_network]
+# (StrOpt) String of physical ethernet name.
+storage_ethernet_name = eth2
+# (StrOpt) String of <ip_addr_start>:<ip_addr_end>
+# tuples specifying the storage network.
+# storage_network_ranges = 3.3.3.3:3.3.3.20
+storage_network_ranges = 3.3.3.1:3.3.3.2
 
 [neutron_client]
 # In order to check dhcp, we need to initialize a neutronclient.

steth/stethclient/agent_api.py

@@ -19,6 +19,7 @@ import sys
 
 from cliff.command import Command
 from cliff.lister import Lister
+from oslo_config import cfg
 from steth.stethclient import utils
 from steth.stethclient.utils import Logger
 from steth.stethclient.utils import setup_server
@@ -261,33 +262,36 @@ class PrintAgentsInfo(Lister):
         parser = super(PrintAgentsInfo, self).get_parser(prog_name)
         return parser
 
+    @utils.timeout(2)
     def is_agent_active(self, agent):
         server = setup_server(agent)
         try:
             server.say_hello()
             return 0
-        except:
+        except utils.TimeoutError:
-            # If this agent is down, "Connection refused" will happen.
+            # If this agent is down, "Connection timed out" will happen.
+            # So we return 1 to set this agent is down.
             return 1
 
     def take_action(self, parsed_args):
-        try:
+        from steth.stethclient.constants import MGMT_AGENTS_CONFIG
-            from steth.stethclient.constants import MGMT_AGENTS_INFOS
+        from steth.stethclient.constants import NET_AGENTS_CONFIG
-            from steth.stethclient.constants import NET_AGENTS_INFOS
+        from steth.stethclient.constants import STORAGE_AGENTS_CONFIG
-            from steth.stethclient.constants import STORAGE_AGENTS_INFOS
-        except Exception as e:
-            Logger.log_fail("Import configure file fail. Because: %s!" % e)
-            sys.exit()
 
         results = []
-        for agent in MGMT_AGENTS_INFOS.keys():
+        for m in range(len(MGMT_AGENTS_CONFIG)):
-            r = []
+            res = []
-            r.append(agent)
+            index = MGMT_AGENTS_CONFIG[m]
-            r.append(MGMT_AGENTS_INFOS[agent])
+            try:
-            r.append(NET_AGENTS_INFOS[agent])
+                node_name = cfg.CONF.node_name_prefix + index
-            r.append(STORAGE_AGENTS_INFOS[agent])
+                res.append(node_name)
-            agent_status = ACTIVE if not self.is_agent_active(agent) else DOWN
+                res.append(index)
-            r.append(agent_status)
+                res.append(NET_AGENTS_CONFIG[m])
-            results.append(r)
+                res.append(STORAGE_AGENTS_CONFIG[m])
+                status = ACTIVE if not self.is_agent_active(index) else DOWN
+                res.append(status)
+            except IndexError:
+                res.append('x')
+            results.append(res)
         return (('Agent Name', 'Management IP', 'Network IP', 'Storage IP',
                 'Alive'), results)

steth/stethclient/constants.py

@@ -16,13 +16,20 @@
 import os
 import socket
 import sys
+from netaddr import iter_iprange
 from oslo_config import cfg
+from steth.stethclient.utils import Logger
+
+
+MGMT_TYPE = 'mgmt'
+NET_TYPE = 'net'
+STORAGE_TYPE = 'storage'
+MGMT_AGENTS_CONFIG = {}
+NET_AGENTS_CONFIG = {}
+STORAGE_AGENTS_CONFIG = {}
+
 
 OPTS = [
-    cfg.ListOpt('network_types', default=[],
-                help="Mappings of network types and prefix of networks."),
-    cfg.ListOpt('nodes_id', default=[],
-                help="List of nodes."),
     cfg.StrOpt('node_name_prefix', default='server-',
                help="Prefix of every node."),
 ]
@@ -38,9 +45,39 @@ NEUTRON_CLIENT_OPTS = [
                help='To get neutronclient, you must specify a auth_url'),
 ]
 
-MGMT_AGENTS_INFOS = {}
+mgmt_network_opts = [
-NET_AGENTS_INFOS = {}
+    cfg.StrOpt('mgmt_ethernet_name',
-STORAGE_AGENTS_INFOS = {}
+               default='lo',
+               help=("Name of managment ethernet name.")),
+    cfg.StrOpt('mgmt_network_ranges',
+               default='1.1.1.1:1.1.1.10',
+               help=("String of <ip_addr_start>:<ip_addr_end> "
+                     "tuples specifying the managment network.")),
+]
+
+sdn_network_opts = [
+    cfg.StrOpt('sdn_ethernet_name',
+               default='lo',
+               help=("Name of sdn ethernet name.")),
+    cfg.StrOpt('sdn_network_ranges',
+               default='2.2.2.2:2.2.2.10',
+               help=("String of <ip_addr_start>:<ip_addr_end> "
+                     "tuples specifying the managment network.")),
+]
+
+storage_network_opts = [
+    cfg.StrOpt('storage_ethernet_name',
+               default='lo',
+               help=("Name of storage ethernet name.")),
+    cfg.StrOpt('storage_network_ranges',
+               default='3.3.3.3:3.3.3.10',
+               help=("String of <ip_addr_start>:<ip_addr_end> "
+                     "tuples specifying the managment network.")),
+]
+
+cfg.CONF.register_opts(mgmt_network_opts, "mgmt_type_network")
+cfg.CONF.register_opts(sdn_network_opts, "sdn_type_network")
+cfg.CONF.register_opts(storage_network_opts, "storage_type_network")
 
 
 ROOTDIR = os.path.dirname(__file__)
@@ -66,13 +103,10 @@ except:
     cfg.CONF([], project='steth',
              default_config_files=[steth_config_file])
 
-MGMT_TYPE = 'mgmt'
-NET_TYPE = 'net'
-STORAGE_TYPE = 'storage'
 
-MGMT_INTERFACE = None
+MGMT_INTERFACE = cfg.CONF.mgmt_type_network.mgmt_ethernet_name
-NET_INTERFACE = None
+NET_INTERFACE = cfg.CONF.sdn_type_network.sdn_ethernet_name
-STORAGE_INTERFACE = None
+STORAGE_INTERFACE = cfg.CONF.storage_type_network.storage_ethernet_name
 
 
 def is_ip(addr):
@@ -85,39 +119,62 @@ def is_ip(addr):
         return 1
 
 
-def check_ip_and_fill(agent_type, net_prefix):
+def get_ip_range(start, end):
-    d = {}
+    generator = iter_iprange(start, end, step=1)
-    name_prefix = cfg.CONF.node_name_prefix
+    ips = []
-    for node in cfg.CONF.nodes_id:
+    while True:
-        if not is_ip(net_prefix + node):
+        try:
-            d[name_prefix + node] = net_prefix + node
+            ips.append(str(generator.next()))
-            agent_type.update(d)
+        except StopIteration:
-        else:
+            break
-            print("%s is not IP!" % name_prefix + node)
+    return ips
+
+
+def program_exits_by_invalid_config():
+    msg = ("Program exits because of invalid config.")
+    Logger.log_high(msg)
+    sys.exit()
+
+
+def check_and_fill_info():
+    infos = cfg.CONF.mgmt_type_network.mgmt_network_ranges
+    if len(infos.split(':')) != 2:
+        program_exits_by_invalid_config()
+    start = infos.split(':')[0]
+    end = infos.split(':')[1]
+    if is_ip(start) or is_ip(end):
+        program_exits_by_invalid_config()
+    global MGMT_AGENTS_CONFIG
+    MGMT_AGENTS_CONFIG = get_ip_range(start, end)
+
+
+def check_and_fill_sdn_info():
+    infos = cfg.CONF.sdn_type_network.sdn_network_ranges
+    if len(infos.split(':')) != 2:
+        program_exits_by_invalid_config()
+    start = infos.split(':')[0]
+    end = infos.split(':')[1]
+    if is_ip(start) or is_ip(end):
+        program_exits_by_invalid_config()
+    global NET_AGENTS_CONFIG
+    NET_AGENTS_CONFIG = get_ip_range(start, end)
+
+
+def check_and_fill_storage_info():
+    infos = cfg.CONF.storage_type_network.storage_network_ranges
+    if len(infos.split(':')) != 2:
+        program_exits_by_invalid_config()
+    start = infos.split(':')[0]
+    end = infos.split(':')[1]
+    if is_ip(start) or is_ip(end):
+        program_exits_by_invalid_config()
+    global STORAGE_AGENTS_CONFIG
+    STORAGE_AGENTS_CONFIG = get_ip_range(start, end)
 
 
 def validate_and_parse_network_types():
-    if not cfg.CONF.network_types:
+    check_and_fill_info()
-        print("You must fill network_types in config file!")
+    check_and_fill_sdn_info()
-        sys.exit()
+    check_and_fill_storage_info()
-    for network_type in cfg.CONF.network_types:
-        net_type, net_interface, net_prefix = network_type.split(':')
-        # parse mgmt networks
-        if net_type == MGMT_TYPE:
-            check_ip_and_fill(MGMT_AGENTS_INFOS, net_prefix)
-            global MGMT_INTERFACE
-            MGMT_INTERFACE = net_interface
-        # parse net networks
-        elif net_type == NET_TYPE:
-            check_ip_and_fill(NET_AGENTS_INFOS, net_prefix)
-            global NET_INTERFACE
-            NET_INTERFACE = net_interface
-        # parse stor networks
-        elif net_type == STORAGE_TYPE:
-            check_ip_and_fill(STORAGE_AGENTS_INFOS, net_prefix)
-            global STORAGE_INTERFACE
-            STORAGE_INTERFACE = net_interface
-        else:
-            print("Unkown network_types: %s" % network_type)
 
 validate_and_parse_network_types()

steth/stethclient/drivers/iperf_api.py

@@ -27,12 +27,12 @@ from steth.stethclient.utils import Logger
 from steth.stethclient.utils import setup_server
 
 try:
-    from steth.stethclient.constants import MGMT_AGENTS_INFOS
+    from steth.stethclient.constants import MGMT_AGENTS_CONFIG
-    from steth.stethclient.constants import NET_AGENTS_INFOS
+    from steth.stethclient.constants import NET_AGENTS_CONFIG
-    from steth.stethclient.constants import STORAGE_AGENTS_INFOS
+    from steth.stethclient.constants import STORAGE_AGENTS_CONFIG
 except:
     Logger.log_fail("Import configure file fail.")
-    MGMT_AGENTS_INFOS = NET_AGENTS_INFOS = STORAGE_AGENTS_INFOS = {
+    MGMT_AGENTS_CONFIG = NET_AGENTS_CONFIG = STORAGE_AGENTS_CONFIG = {
         'agent-64': "127.0.0.1",
         'agent-65': "127.0.0.1",
     }
@@ -120,9 +120,9 @@ class CheckIperf(Lister):
             return (('Field', 'Value'),
                     ((k, v) for k, v in res['data'].items()))
         elif parsed_args.iperf_server_type == 'others':
-            mgmt_host = MGMT_AGENTS_INFOS[parsed_args.server_agent]
+            mgmt_host = MGMT_AGENTS_CONFIG[parsed_args.server_agent]
-            net_host = NET_AGENTS_INFOS[parsed_args.server_agent]
+            net_host = NET_AGENTS_CONFIG[parsed_args.server_agent]
-            storage_host = STORAGE_AGENTS_INFOS[parsed_args.server_agent]
+            storage_host = STORAGE_AGENTS_CONFIG[parsed_args.server_agent]
             bandwidth = parsed_args.client_bandwidth
             mgmt_res = self.take_iperf_client(
                 client=client,

steth/stethclient/utils.py

@@ -13,8 +13,12 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import errno
+from functools import wraps
 import logging
 import jsonrpclib
+import os
+import signal
 import six
 import socket
 import sys
@@ -81,32 +85,15 @@ class Logger():
 
 LISTEN_PORT = 9698
 
-try:
-    from steth.stethclient.constants import MGMT_AGENTS_INFOS
-    from steth.stethclient.constants import NET_AGENTS_INFOS
-    from steth.stethclient.constants import STORAGE_AGENTS_INFOS
-except:
-    Logger.log_fail("Import configure file fail.")
-    MGMT_AGENTS_INFOS = NET_AGENTS_INFOS = STORAGE_AGENTS_INFOS = {
-        'agent-64': "127.0.0.1",
-        'agent-65': "127.0.0.1",
-    }
-
 
 def setup_server(agent):
     log = logging.getLogger(__name__)
-    if agent in MGMT_AGENTS_INFOS:
-        log.debug('get agent:%s ip_address:%s' % (
-            agent, MGMT_AGENTS_INFOS[agent]))
-    else:
-        log.error('Agent %s not configured. Please check it.' % (agent))
-        sys.exit()
     log.debug('Begin create connection with http://%s:%s.' % (
         agent,
         LISTEN_PORT))
     server = jsonrpclib.Server('http://%s:%s' %
-                               (MGMT_AGENTS_INFOS[agent], LISTEN_PORT))
+                               (agent, LISTEN_PORT))
-    log.debug('Create connection with %s success.' % (agent))
+    log.debug('Create connection with %s success.' % agent)
     return server
 
 
@@ -124,13 +111,16 @@ def get_ip_from_agent(node, net_type):
     from steth.stethclient.constants import MGMT_TYPE
     from steth.stethclient.constants import NET_TYPE
     from steth.stethclient.constants import STORAGE_TYPE
+    from steth.stethclient.constants import MGMT_AGENTS_CONFIG
+    from steth.stethclient.constants import NET_AGENTS_CONFIG
+    from steth.stethclient.constants import STORAGE_AGENTS_CONFIG
     try:
         if net_type == NET_TYPE:
-            return NET_AGENTS_INFOS[node]
+            return NET_AGENTS_CONFIG[node]
         elif net_type == MGMT_TYPE:
-            return MGMT_AGENTS_INFOS[node]
+            return MGMT_AGENTS_CONFIG[node]
         elif net_type == STORAGE_TYPE:
-            return STORAGE_AGENTS_INFOS[node]
+            return STORAGE_AGENTS_CONFIG[node]
         else:
             return 1
     except Exception as e:
@@ -156,3 +146,26 @@ def is_uuid_like(val):
         return str(uuid.UUID(val)).replace('-', '') == _format_uuid_string(val)
     except (TypeError, ValueError, AttributeError):
         return False
+
+
+class TimeoutError(Exception):
+    pass
+
+
+def timeout(seconds=10, error_message=os.strerror(errno.ETIME)):
+    def decorator(func):
+        def _handle_timeout(signum, frame):
+            raise TimeoutError(error_message)
+
+        def wrapper(*args, **kwargs):
+            signal.signal(signal.SIGALRM, _handle_timeout)
+            signal.alarm(seconds)
+            try:
+                result = func(*args, **kwargs)
+            finally:
+                signal.alarm(0)
+            return result
+
+        return wraps(func)(wrapper)
+
+    return decorator