From 89666323f7e4d0efd3d607155337711f83a2ff30 Mon Sep 17 00:00:00 2001 From: changzhi Date: Thu, 4 Aug 2016 17:29:37 +0800 Subject: [PATCH] Refactor read config file In order to make configuration more clearly, seperating "network_types" configuration into serveral configuration. Add timeout decorator to make sure program can goes more smoothly. Change-Id: I0d9765b23f89e2c8ac8b1d9047cfebb89f04df8d --- etc/steth.conf | 40 ++++--- steth/stethclient/agent_api.py | 40 ++++--- steth/stethclient/constants.py | 145 +++++++++++++++++-------- steth/stethclient/drivers/iperf_api.py | 14 +-- steth/stethclient/utils.py | 57 ++++++---- 5 files changed, 189 insertions(+), 107 deletions(-) diff --git a/etc/steth.conf b/etc/steth.conf index 80b3e76..9ba7754 100644 --- a/etc/steth.conf +++ b/etc/steth.conf @@ -1,26 +1,34 @@ [DEFAULT] -# (ListOpt) list of networks types. -# We may have multi network types in one node, such as mgmt, net and stroage. -# so this value should be a list. -# We seperate each item by ":". Treat first item as network type. -# The second is physical nic name. And the third is network_prefix. -# Example: "mgmt:eth0:1.1.1.,net:eth1:2.2.2.,storage:eth2:3.3.3." -network_types=mgmt:eth0:1.1.1.,net:eth1:2.2.2.,storage:eth2:3.3.3. - -# (ListOpt) All nodes info. Just need sequence number. -# Example: 64, 65, 66 -nodes_id=39,233,64,65,66 - -# (StrOpt) Name prefix of every node. By default, this value +#(StrOpt) Name prefix of every node. By default, this value # is "server". We combine "node_name_prefix" with # "nodes_id", to define nodes. Such as "server-64", "server-68" # and so on. In every region, we give every node a specific name. # Ensure that DNS can resolve the nodes. node_name_prefix=server- -# (StrOpt) Name of physical interface in each network node or -# compute node. -physical_interface=eth3 +[mgmt_type_network] +# (StrOpt) String of physical ethernet name. +mgmt_ethernet_name = eth0 +# (StrOpt) String of : +# tuples specifying the managment network. +# mgmt_network_ranges = eth0:1.1.1.1:1.1.1.10 +mgmt_network_ranges = 1.1.1.1:1.1.1.2 + +[sdn_type_network] +# (StrOpt) String of physical ethernet name. +sdn_ethernet_name = eth1 +# (StrOpt) String of : +# tuples specifying the sdn network. +# sdn_network_ranges = 2.2.2.2:2.2.2.10 +sdn_network_ranges = 2.2.2.1:2.2.2.2 + +[storage_type_network] +# (StrOpt) String of physical ethernet name. +storage_ethernet_name = eth2 +# (StrOpt) String of : +# tuples specifying the storage network. +# storage_network_ranges = 3.3.3.3:3.3.3.20 +storage_network_ranges = 3.3.3.1:3.3.3.2 [neutron_client] # In order to check dhcp, we need to initialize a neutronclient. diff --git a/steth/stethclient/agent_api.py b/steth/stethclient/agent_api.py index 3667de6..80f8259 100644 --- a/steth/stethclient/agent_api.py +++ b/steth/stethclient/agent_api.py @@ -19,6 +19,7 @@ import sys from cliff.command import Command from cliff.lister import Lister +from oslo_config import cfg from steth.stethclient import utils from steth.stethclient.utils import Logger from steth.stethclient.utils import setup_server @@ -261,33 +262,36 @@ class PrintAgentsInfo(Lister): parser = super(PrintAgentsInfo, self).get_parser(prog_name) return parser + @utils.timeout(2) def is_agent_active(self, agent): server = setup_server(agent) try: server.say_hello() return 0 - except: - # If this agent is down, "Connection refused" will happen. + except utils.TimeoutError: + # If this agent is down, "Connection timed out" will happen. + # So we return 1 to set this agent is down. return 1 def take_action(self, parsed_args): - try: - from steth.stethclient.constants import MGMT_AGENTS_INFOS - from steth.stethclient.constants import NET_AGENTS_INFOS - from steth.stethclient.constants import STORAGE_AGENTS_INFOS - except Exception as e: - Logger.log_fail("Import configure file fail. Because: %s!" % e) - sys.exit() + from steth.stethclient.constants import MGMT_AGENTS_CONFIG + from steth.stethclient.constants import NET_AGENTS_CONFIG + from steth.stethclient.constants import STORAGE_AGENTS_CONFIG results = [] - for agent in MGMT_AGENTS_INFOS.keys(): - r = [] - r.append(agent) - r.append(MGMT_AGENTS_INFOS[agent]) - r.append(NET_AGENTS_INFOS[agent]) - r.append(STORAGE_AGENTS_INFOS[agent]) - agent_status = ACTIVE if not self.is_agent_active(agent) else DOWN - r.append(agent_status) - results.append(r) + for m in range(len(MGMT_AGENTS_CONFIG)): + res = [] + index = MGMT_AGENTS_CONFIG[m] + try: + node_name = cfg.CONF.node_name_prefix + index + res.append(node_name) + res.append(index) + res.append(NET_AGENTS_CONFIG[m]) + res.append(STORAGE_AGENTS_CONFIG[m]) + status = ACTIVE if not self.is_agent_active(index) else DOWN + res.append(status) + except IndexError: + res.append('x') + results.append(res) return (('Agent Name', 'Management IP', 'Network IP', 'Storage IP', 'Alive'), results) diff --git a/steth/stethclient/constants.py b/steth/stethclient/constants.py index 61a3efc..74d3148 100644 --- a/steth/stethclient/constants.py +++ b/steth/stethclient/constants.py @@ -16,13 +16,20 @@ import os import socket import sys +from netaddr import iter_iprange from oslo_config import cfg +from steth.stethclient.utils import Logger + + +MGMT_TYPE = 'mgmt' +NET_TYPE = 'net' +STORAGE_TYPE = 'storage' +MGMT_AGENTS_CONFIG = {} +NET_AGENTS_CONFIG = {} +STORAGE_AGENTS_CONFIG = {} + OPTS = [ - cfg.ListOpt('network_types', default=[], - help="Mappings of network types and prefix of networks."), - cfg.ListOpt('nodes_id', default=[], - help="List of nodes."), cfg.StrOpt('node_name_prefix', default='server-', help="Prefix of every node."), ] @@ -38,9 +45,39 @@ NEUTRON_CLIENT_OPTS = [ help='To get neutronclient, you must specify a auth_url'), ] -MGMT_AGENTS_INFOS = {} -NET_AGENTS_INFOS = {} -STORAGE_AGENTS_INFOS = {} +mgmt_network_opts = [ + cfg.StrOpt('mgmt_ethernet_name', + default='lo', + help=("Name of managment ethernet name.")), + cfg.StrOpt('mgmt_network_ranges', + default='1.1.1.1:1.1.1.10', + help=("String of : " + "tuples specifying the managment network.")), +] + +sdn_network_opts = [ + cfg.StrOpt('sdn_ethernet_name', + default='lo', + help=("Name of sdn ethernet name.")), + cfg.StrOpt('sdn_network_ranges', + default='2.2.2.2:2.2.2.10', + help=("String of : " + "tuples specifying the managment network.")), +] + +storage_network_opts = [ + cfg.StrOpt('storage_ethernet_name', + default='lo', + help=("Name of storage ethernet name.")), + cfg.StrOpt('storage_network_ranges', + default='3.3.3.3:3.3.3.10', + help=("String of : " + "tuples specifying the managment network.")), +] + +cfg.CONF.register_opts(mgmt_network_opts, "mgmt_type_network") +cfg.CONF.register_opts(sdn_network_opts, "sdn_type_network") +cfg.CONF.register_opts(storage_network_opts, "storage_type_network") ROOTDIR = os.path.dirname(__file__) @@ -66,13 +103,10 @@ except: cfg.CONF([], project='steth', default_config_files=[steth_config_file]) -MGMT_TYPE = 'mgmt' -NET_TYPE = 'net' -STORAGE_TYPE = 'storage' -MGMT_INTERFACE = None -NET_INTERFACE = None -STORAGE_INTERFACE = None +MGMT_INTERFACE = cfg.CONF.mgmt_type_network.mgmt_ethernet_name +NET_INTERFACE = cfg.CONF.sdn_type_network.sdn_ethernet_name +STORAGE_INTERFACE = cfg.CONF.storage_type_network.storage_ethernet_name def is_ip(addr): @@ -85,39 +119,62 @@ def is_ip(addr): return 1 -def check_ip_and_fill(agent_type, net_prefix): - d = {} - name_prefix = cfg.CONF.node_name_prefix - for node in cfg.CONF.nodes_id: - if not is_ip(net_prefix + node): - d[name_prefix + node] = net_prefix + node - agent_type.update(d) - else: - print("%s is not IP!" % name_prefix + node) +def get_ip_range(start, end): + generator = iter_iprange(start, end, step=1) + ips = [] + while True: + try: + ips.append(str(generator.next())) + except StopIteration: + break + return ips + + +def program_exits_by_invalid_config(): + msg = ("Program exits because of invalid config.") + Logger.log_high(msg) + sys.exit() + + +def check_and_fill_info(): + infos = cfg.CONF.mgmt_type_network.mgmt_network_ranges + if len(infos.split(':')) != 2: + program_exits_by_invalid_config() + start = infos.split(':')[0] + end = infos.split(':')[1] + if is_ip(start) or is_ip(end): + program_exits_by_invalid_config() + global MGMT_AGENTS_CONFIG + MGMT_AGENTS_CONFIG = get_ip_range(start, end) + + +def check_and_fill_sdn_info(): + infos = cfg.CONF.sdn_type_network.sdn_network_ranges + if len(infos.split(':')) != 2: + program_exits_by_invalid_config() + start = infos.split(':')[0] + end = infos.split(':')[1] + if is_ip(start) or is_ip(end): + program_exits_by_invalid_config() + global NET_AGENTS_CONFIG + NET_AGENTS_CONFIG = get_ip_range(start, end) + + +def check_and_fill_storage_info(): + infos = cfg.CONF.storage_type_network.storage_network_ranges + if len(infos.split(':')) != 2: + program_exits_by_invalid_config() + start = infos.split(':')[0] + end = infos.split(':')[1] + if is_ip(start) or is_ip(end): + program_exits_by_invalid_config() + global STORAGE_AGENTS_CONFIG + STORAGE_AGENTS_CONFIG = get_ip_range(start, end) def validate_and_parse_network_types(): - if not cfg.CONF.network_types: - print("You must fill network_types in config file!") - sys.exit() - for network_type in cfg.CONF.network_types: - net_type, net_interface, net_prefix = network_type.split(':') - # parse mgmt networks - if net_type == MGMT_TYPE: - check_ip_and_fill(MGMT_AGENTS_INFOS, net_prefix) - global MGMT_INTERFACE - MGMT_INTERFACE = net_interface - # parse net networks - elif net_type == NET_TYPE: - check_ip_and_fill(NET_AGENTS_INFOS, net_prefix) - global NET_INTERFACE - NET_INTERFACE = net_interface - # parse stor networks - elif net_type == STORAGE_TYPE: - check_ip_and_fill(STORAGE_AGENTS_INFOS, net_prefix) - global STORAGE_INTERFACE - STORAGE_INTERFACE = net_interface - else: - print("Unkown network_types: %s" % network_type) + check_and_fill_info() + check_and_fill_sdn_info() + check_and_fill_storage_info() validate_and_parse_network_types() diff --git a/steth/stethclient/drivers/iperf_api.py b/steth/stethclient/drivers/iperf_api.py index 8e1d7d9..619ff08 100644 --- a/steth/stethclient/drivers/iperf_api.py +++ b/steth/stethclient/drivers/iperf_api.py @@ -27,12 +27,12 @@ from steth.stethclient.utils import Logger from steth.stethclient.utils import setup_server try: - from steth.stethclient.constants import MGMT_AGENTS_INFOS - from steth.stethclient.constants import NET_AGENTS_INFOS - from steth.stethclient.constants import STORAGE_AGENTS_INFOS + from steth.stethclient.constants import MGMT_AGENTS_CONFIG + from steth.stethclient.constants import NET_AGENTS_CONFIG + from steth.stethclient.constants import STORAGE_AGENTS_CONFIG except: Logger.log_fail("Import configure file fail.") - MGMT_AGENTS_INFOS = NET_AGENTS_INFOS = STORAGE_AGENTS_INFOS = { + MGMT_AGENTS_CONFIG = NET_AGENTS_CONFIG = STORAGE_AGENTS_CONFIG = { 'agent-64': "127.0.0.1", 'agent-65': "127.0.0.1", } @@ -120,9 +120,9 @@ class CheckIperf(Lister): return (('Field', 'Value'), ((k, v) for k, v in res['data'].items())) elif parsed_args.iperf_server_type == 'others': - mgmt_host = MGMT_AGENTS_INFOS[parsed_args.server_agent] - net_host = NET_AGENTS_INFOS[parsed_args.server_agent] - storage_host = STORAGE_AGENTS_INFOS[parsed_args.server_agent] + mgmt_host = MGMT_AGENTS_CONFIG[parsed_args.server_agent] + net_host = NET_AGENTS_CONFIG[parsed_args.server_agent] + storage_host = STORAGE_AGENTS_CONFIG[parsed_args.server_agent] bandwidth = parsed_args.client_bandwidth mgmt_res = self.take_iperf_client( client=client, diff --git a/steth/stethclient/utils.py b/steth/stethclient/utils.py index 58b763d..54f8cb5 100644 --- a/steth/stethclient/utils.py +++ b/steth/stethclient/utils.py @@ -13,8 +13,12 @@ # License for the specific language governing permissions and limitations # under the License. +import errno +from functools import wraps import logging import jsonrpclib +import os +import signal import six import socket import sys @@ -81,32 +85,15 @@ class Logger(): LISTEN_PORT = 9698 -try: - from steth.stethclient.constants import MGMT_AGENTS_INFOS - from steth.stethclient.constants import NET_AGENTS_INFOS - from steth.stethclient.constants import STORAGE_AGENTS_INFOS -except: - Logger.log_fail("Import configure file fail.") - MGMT_AGENTS_INFOS = NET_AGENTS_INFOS = STORAGE_AGENTS_INFOS = { - 'agent-64': "127.0.0.1", - 'agent-65': "127.0.0.1", - } - def setup_server(agent): log = logging.getLogger(__name__) - if agent in MGMT_AGENTS_INFOS: - log.debug('get agent:%s ip_address:%s' % ( - agent, MGMT_AGENTS_INFOS[agent])) - else: - log.error('Agent %s not configured. Please check it.' % (agent)) - sys.exit() log.debug('Begin create connection with http://%s:%s.' % ( agent, LISTEN_PORT)) server = jsonrpclib.Server('http://%s:%s' % - (MGMT_AGENTS_INFOS[agent], LISTEN_PORT)) - log.debug('Create connection with %s success.' % (agent)) + (agent, LISTEN_PORT)) + log.debug('Create connection with %s success.' % agent) return server @@ -124,13 +111,16 @@ def get_ip_from_agent(node, net_type): from steth.stethclient.constants import MGMT_TYPE from steth.stethclient.constants import NET_TYPE from steth.stethclient.constants import STORAGE_TYPE + from steth.stethclient.constants import MGMT_AGENTS_CONFIG + from steth.stethclient.constants import NET_AGENTS_CONFIG + from steth.stethclient.constants import STORAGE_AGENTS_CONFIG try: if net_type == NET_TYPE: - return NET_AGENTS_INFOS[node] + return NET_AGENTS_CONFIG[node] elif net_type == MGMT_TYPE: - return MGMT_AGENTS_INFOS[node] + return MGMT_AGENTS_CONFIG[node] elif net_type == STORAGE_TYPE: - return STORAGE_AGENTS_INFOS[node] + return STORAGE_AGENTS_CONFIG[node] else: return 1 except Exception as e: @@ -156,3 +146,26 @@ def is_uuid_like(val): return str(uuid.UUID(val)).replace('-', '') == _format_uuid_string(val) except (TypeError, ValueError, AttributeError): return False + + +class TimeoutError(Exception): + pass + + +def timeout(seconds=10, error_message=os.strerror(errno.ETIME)): + def decorator(func): + def _handle_timeout(signum, frame): + raise TimeoutError(error_message) + + def wrapper(*args, **kwargs): + signal.signal(signal.SIGALRM, _handle_timeout) + signal.alarm(seconds) + try: + result = func(*args, **kwargs) + finally: + signal.alarm(0) + return result + + return wraps(func)(wrapper) + + return decorator