x/steth
Code Issues Proposed changes

Refactor read config file

Browse Source 
In order to make configuration more clearly,
seperating "network_types" configuration into serveral configuration.

Add timeout decorator to make sure program can goes more smoothly.

Change-Id: I0d9765b23f89e2c8ac8b1d9047cfebb89f04df8d
This commit is contained in:
changzhi 2016-08-04 17:29:37 +08:00
parent db238051ce
commit 89666323f7
5 changed files with 189 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
etc/steth.conf
View File

@ -1,16 +1,4 @@
[DEFAULT] [DEFAULT]
# (ListOpt) list of networks types.
# We may have multi network types in one node, such as mgmt, net and stroage.
# so this value should be a list.
# We seperate each item by ":". Treat first item as network type.
# The second is physical nic name. And the third is network_prefix.
# Example: "mgmt:eth0:1.1.1.,net:eth1:2.2.2.,storage:eth2:3.3.3."
network_types=mgmt:eth0:1.1.1.,net:eth1:2.2.2.,storage:eth2:3.3.3.
# (ListOpt) All nodes info. Just need sequence number.
# Example: 64, 65, 66
nodes_id=39,233,64,65,66
#(StrOpt) Name prefix of every node. By default, this value #(StrOpt) Name prefix of every node. By default, this value
# is "server". We combine "node_name_prefix" with # is "server". We combine "node_name_prefix" with
# "nodes_id", to define nodes. Such as "server-64", "server-68" # "nodes_id", to define nodes. Such as "server-64", "server-68"
@ -18,9 +6,29 @@ nodes_id=39,233,64,65,66
# Ensure that DNS can resolve the nodes. # Ensure that DNS can resolve the nodes.
node_name_prefix=server- node_name_prefix=server-
# (StrOpt) Name of physical interface in each network node or [mgmt_type_network]
# compute node. # (StrOpt) String of physical ethernet name.
physical_interface=eth3 mgmt_ethernet_name = eth0
# (StrOpt) String of <ip_addr_start>:<ip_addr_end>
# tuples specifying the managment network.
# mgmt_network_ranges = eth0:1.1.1.1:1.1.1.10
mgmt_network_ranges = 1.1.1.1:1.1.1.2
[sdn_type_network]
# (StrOpt) String of physical ethernet name.
sdn_ethernet_name = eth1
# (StrOpt) String of <ip_addr_start>:<ip_addr_end>
# tuples specifying the sdn network.
# sdn_network_ranges = 2.2.2.2:2.2.2.10
sdn_network_ranges = 2.2.2.1:2.2.2.2
[storage_type_network]
# (StrOpt) String of physical ethernet name.
storage_ethernet_name = eth2
# (StrOpt) String of <ip_addr_start>:<ip_addr_end>
# tuples specifying the storage network.
# storage_network_ranges = 3.3.3.3:3.3.3.20
storage_network_ranges = 3.3.3.1:3.3.3.2
[neutron_client] [neutron_client]
# In order to check dhcp, we need to initialize a neutronclient. # In order to check dhcp, we need to initialize a neutronclient.

40
steth/stethclient/agent_api.py
View File

@ -19,6 +19,7 @@ import sys
from cliff.command import Command from cliff.command import Command
from cliff.lister import Lister from cliff.lister import Lister
from oslo_config import cfg
from steth.stethclient import utils from steth.stethclient import utils
from steth.stethclient.utils import Logger from steth.stethclient.utils import Logger
from steth.stethclient.utils import setup_server from steth.stethclient.utils import setup_server
@ -261,33 +262,36 @@ class PrintAgentsInfo(Lister):
parser = super(PrintAgentsInfo, self).get_parser(prog_name) parser = super(PrintAgentsInfo, self).get_parser(prog_name)
return parser return parser
@utils.timeout(2)
def is_agent_active(self, agent): def is_agent_active(self, agent):
server = setup_server(agent) server = setup_server(agent)
try: try:
server.say_hello() server.say_hello()
return 0 return 0
except: except utils.TimeoutError:
# If this agent is down, "Connection refused" will happen. # If this agent is down, "Connection timed out" will happen.
# So we return 1 to set this agent is down.
return 1 return 1
def take_action(self, parsed_args): def take_action(self, parsed_args):
try: from steth.stethclient.constants import MGMT_AGENTS_CONFIG
from steth.stethclient.constants import MGMT_AGENTS_INFOS from steth.stethclient.constants import NET_AGENTS_CONFIG
from steth.stethclient.constants import NET_AGENTS_INFOS from steth.stethclient.constants import STORAGE_AGENTS_CONFIG
from steth.stethclient.constants import STORAGE_AGENTS_INFOS
except Exception as e:
Logger.log_fail("Import configure file fail. Because: %s!" % e)
sys.exit()
results = [] results = []
for agent in MGMT_AGENTS_INFOS.keys(): for m in range(len(MGMT_AGENTS_CONFIG)):
r = [] res = []
r.append(agent) index = MGMT_AGENTS_CONFIG[m]
r.append(MGMT_AGENTS_INFOS[agent]) try:
r.append(NET_AGENTS_INFOS[agent]) node_name = cfg.CONF.node_name_prefix + index
r.append(STORAGE_AGENTS_INFOS[agent]) res.append(node_name)
agent_status = ACTIVE if not self.is_agent_active(agent) else DOWN res.append(index)
r.append(agent_status) res.append(NET_AGENTS_CONFIG[m])
results.append(r) res.append(STORAGE_AGENTS_CONFIG[m])
status = ACTIVE if not self.is_agent_active(index) else DOWN
res.append(status)
except IndexError:
res.append('x')
results.append(res)
return (('Agent Name', 'Management IP', 'Network IP', 'Storage IP', return (('Agent Name', 'Management IP', 'Network IP', 'Storage IP',
'Alive'), results) 'Alive'), results)

145
steth/stethclient/constants.py
View File

@ -16,13 +16,20 @@
import os import os
import socket import socket
import sys import sys
from netaddr import iter_iprange
from oslo_config import cfg from oslo_config import cfg
from steth.stethclient.utils import Logger
MGMT_TYPE = 'mgmt'
NET_TYPE = 'net'
STORAGE_TYPE = 'storage'
MGMT_AGENTS_CONFIG = {}
NET_AGENTS_CONFIG = {}
STORAGE_AGENTS_CONFIG = {}
OPTS = [ OPTS = [
cfg.ListOpt('network_types', default=[],
help="Mappings of network types and prefix of networks."),
cfg.ListOpt('nodes_id', default=[],
help="List of nodes."),
cfg.StrOpt('node_name_prefix', default='server-', cfg.StrOpt('node_name_prefix', default='server-',
help="Prefix of every node."), help="Prefix of every node."),
] ]
@ -38,9 +45,39 @@ NEUTRON_CLIENT_OPTS = [
help='To get neutronclient, you must specify a auth_url'), help='To get neutronclient, you must specify a auth_url'),
] ]
MGMT_AGENTS_INFOS = {} mgmt_network_opts = [
NET_AGENTS_INFOS = {} cfg.StrOpt('mgmt_ethernet_name',
STORAGE_AGENTS_INFOS = {} default='lo',
help=("Name of managment ethernet name.")),
cfg.StrOpt('mgmt_network_ranges',
default='1.1.1.1:1.1.1.10',
help=("String of <ip_addr_start>:<ip_addr_end> "
"tuples specifying the managment network.")),
]
sdn_network_opts = [
cfg.StrOpt('sdn_ethernet_name',
default='lo',
help=("Name of sdn ethernet name.")),
cfg.StrOpt('sdn_network_ranges',
default='2.2.2.2:2.2.2.10',
help=("String of <ip_addr_start>:<ip_addr_end> "
"tuples specifying the managment network.")),
]
storage_network_opts = [
cfg.StrOpt('storage_ethernet_name',
default='lo',
help=("Name of storage ethernet name.")),
cfg.StrOpt('storage_network_ranges',
default='3.3.3.3:3.3.3.10',
help=("String of <ip_addr_start>:<ip_addr_end> "
"tuples specifying the managment network.")),
]
cfg.CONF.register_opts(mgmt_network_opts, "mgmt_type_network")
cfg.CONF.register_opts(sdn_network_opts, "sdn_type_network")
cfg.CONF.register_opts(storage_network_opts, "storage_type_network")
ROOTDIR = os.path.dirname(__file__) ROOTDIR = os.path.dirname(__file__)
@ -66,13 +103,10 @@ except:
cfg.CONF([], project='steth', cfg.CONF([], project='steth',
default_config_files=[steth_config_file]) default_config_files=[steth_config_file])
MGMT_TYPE = 'mgmt'
NET_TYPE = 'net'
STORAGE_TYPE = 'storage'
MGMT_INTERFACE = None MGMT_INTERFACE = cfg.CONF.mgmt_type_network.mgmt_ethernet_name
NET_INTERFACE = None NET_INTERFACE = cfg.CONF.sdn_type_network.sdn_ethernet_name
STORAGE_INTERFACE = None STORAGE_INTERFACE = cfg.CONF.storage_type_network.storage_ethernet_name
def is_ip(addr): def is_ip(addr):
@ -85,39 +119,62 @@ def is_ip(addr):
return 1 return 1
def check_ip_and_fill(agent_type, net_prefix): def get_ip_range(start, end):
d = {} generator = iter_iprange(start, end, step=1)
name_prefix = cfg.CONF.node_name_prefix ips = []
for node in cfg.CONF.nodes_id: while True:
if not is_ip(net_prefix + node): try:
d[name_prefix + node] = net_prefix + node ips.append(str(generator.next()))
agent_type.update(d) except StopIteration:
else: break
print("%s is not IP!" % name_prefix + node) return ips
def program_exits_by_invalid_config():
msg = ("Program exits because of invalid config.")
Logger.log_high(msg)
sys.exit()
def check_and_fill_info():
infos = cfg.CONF.mgmt_type_network.mgmt_network_ranges
if len(infos.split(':')) != 2:
program_exits_by_invalid_config()
start = infos.split(':')[0]
end = infos.split(':')[1]
if is_ip(start) or is_ip(end):
program_exits_by_invalid_config()
global MGMT_AGENTS_CONFIG
MGMT_AGENTS_CONFIG = get_ip_range(start, end)
def check_and_fill_sdn_info():
infos = cfg.CONF.sdn_type_network.sdn_network_ranges
if len(infos.split(':')) != 2:
program_exits_by_invalid_config()
start = infos.split(':')[0]
end = infos.split(':')[1]
if is_ip(start) or is_ip(end):
program_exits_by_invalid_config()
global NET_AGENTS_CONFIG
NET_AGENTS_CONFIG = get_ip_range(start, end)
def check_and_fill_storage_info():
infos = cfg.CONF.storage_type_network.storage_network_ranges
if len(infos.split(':')) != 2:
program_exits_by_invalid_config()
start = infos.split(':')[0]
end = infos.split(':')[1]
if is_ip(start) or is_ip(end):
program_exits_by_invalid_config()
global STORAGE_AGENTS_CONFIG
STORAGE_AGENTS_CONFIG = get_ip_range(start, end)
def validate_and_parse_network_types(): def validate_and_parse_network_types():
if not cfg.CONF.network_types: check_and_fill_info()
print("You must fill network_types in config file!") check_and_fill_sdn_info()
sys.exit() check_and_fill_storage_info()
for network_type in cfg.CONF.network_types:
net_type, net_interface, net_prefix = network_type.split(':')
# parse mgmt networks
if net_type == MGMT_TYPE:
check_ip_and_fill(MGMT_AGENTS_INFOS, net_prefix)
global MGMT_INTERFACE
MGMT_INTERFACE = net_interface
# parse net networks
elif net_type == NET_TYPE:
check_ip_and_fill(NET_AGENTS_INFOS, net_prefix)
global NET_INTERFACE
NET_INTERFACE = net_interface
# parse stor networks
elif net_type == STORAGE_TYPE:
check_ip_and_fill(STORAGE_AGENTS_INFOS, net_prefix)
global STORAGE_INTERFACE
STORAGE_INTERFACE = net_interface
else:
print("Unkown network_types: %s" % network_type)
validate_and_parse_network_types() validate_and_parse_network_types()

14
steth/stethclient/drivers/iperf_api.py
View File

@ -27,12 +27,12 @@ from steth.stethclient.utils import Logger
from steth.stethclient.utils import setup_server from steth.stethclient.utils import setup_server
try: try:
from steth.stethclient.constants import MGMT_AGENTS_INFOS from steth.stethclient.constants import MGMT_AGENTS_CONFIG
from steth.stethclient.constants import NET_AGENTS_INFOS from steth.stethclient.constants import NET_AGENTS_CONFIG
from steth.stethclient.constants import STORAGE_AGENTS_INFOS from steth.stethclient.constants import STORAGE_AGENTS_CONFIG
except: except:
Logger.log_fail("Import configure file fail.") Logger.log_fail("Import configure file fail.")
MGMT_AGENTS_INFOS = NET_AGENTS_INFOS = STORAGE_AGENTS_INFOS = { MGMT_AGENTS_CONFIG = NET_AGENTS_CONFIG = STORAGE_AGENTS_CONFIG = {
'agent-64': "127.0.0.1", 'agent-64': "127.0.0.1",
'agent-65': "127.0.0.1", 'agent-65': "127.0.0.1",
} }
@ -120,9 +120,9 @@ class CheckIperf(Lister):
return (('Field', 'Value'), return (('Field', 'Value'),
((k, v) for k, v in res['data'].items())) ((k, v) for k, v in res['data'].items()))
elif parsed_args.iperf_server_type == 'others': elif parsed_args.iperf_server_type == 'others':
mgmt_host = MGMT_AGENTS_INFOS[parsed_args.server_agent] mgmt_host = MGMT_AGENTS_CONFIG[parsed_args.server_agent]
net_host = NET_AGENTS_INFOS[parsed_args.server_agent] net_host = NET_AGENTS_CONFIG[parsed_args.server_agent]
storage_host = STORAGE_AGENTS_INFOS[parsed_args.server_agent] storage_host = STORAGE_AGENTS_CONFIG[parsed_args.server_agent]
bandwidth = parsed_args.client_bandwidth bandwidth = parsed_args.client_bandwidth
mgmt_res = self.take_iperf_client( mgmt_res = self.take_iperf_client(
client=client, client=client,

57
steth/stethclient/utils.py
View File

@ -13,8 +13,12 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
import errno
from functools import wraps
import logging import logging
import jsonrpclib import jsonrpclib
import os
import signal
import six import six
import socket import socket
import sys import sys
@ -81,32 +85,15 @@ class Logger():
LISTEN_PORT = 9698 LISTEN_PORT = 9698
try:
from steth.stethclient.constants import MGMT_AGENTS_INFOS
from steth.stethclient.constants import NET_AGENTS_INFOS
from steth.stethclient.constants import STORAGE_AGENTS_INFOS
except:
Logger.log_fail("Import configure file fail.")
MGMT_AGENTS_INFOS = NET_AGENTS_INFOS = STORAGE_AGENTS_INFOS = {
'agent-64': "127.0.0.1",
'agent-65': "127.0.0.1",
}
def setup_server(agent): def setup_server(agent):
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
if agent in MGMT_AGENTS_INFOS:
log.debug('get agent:%s ip_address:%s' % (
agent, MGMT_AGENTS_INFOS[agent]))
else:
log.error('Agent %s not configured. Please check it.' % (agent))
sys.exit()
log.debug('Begin create connection with http://%s:%s.' % ( log.debug('Begin create connection with http://%s:%s.' % (
agent, agent,
LISTEN_PORT)) LISTEN_PORT))
server = jsonrpclib.Server('http://%s:%s' % server = jsonrpclib.Server('http://%s:%s' %
(MGMT_AGENTS_INFOS[agent], LISTEN_PORT)) (agent, LISTEN_PORT))
log.debug('Create connection with %s success.' % (agent)) log.debug('Create connection with %s success.' % agent)
return server return server
@ -124,13 +111,16 @@ def get_ip_from_agent(node, net_type):
from steth.stethclient.constants import MGMT_TYPE from steth.stethclient.constants import MGMT_TYPE
from steth.stethclient.constants import NET_TYPE from steth.stethclient.constants import NET_TYPE
from steth.stethclient.constants import STORAGE_TYPE from steth.stethclient.constants import STORAGE_TYPE
from steth.stethclient.constants import MGMT_AGENTS_CONFIG
from steth.stethclient.constants import NET_AGENTS_CONFIG
from steth.stethclient.constants import STORAGE_AGENTS_CONFIG
try: try:
if net_type == NET_TYPE: if net_type == NET_TYPE:
return NET_AGENTS_INFOS[node] return NET_AGENTS_CONFIG[node]
elif net_type == MGMT_TYPE: elif net_type == MGMT_TYPE:
return MGMT_AGENTS_INFOS[node] return MGMT_AGENTS_CONFIG[node]
elif net_type == STORAGE_TYPE: elif net_type == STORAGE_TYPE:
return STORAGE_AGENTS_INFOS[node] return STORAGE_AGENTS_CONFIG[node]
else: else:
return 1 return 1
except Exception as e: except Exception as e:
@ -156,3 +146,26 @@ def is_uuid_like(val):
return str(uuid.UUID(val)).replace('-', '') == _format_uuid_string(val) return str(uuid.UUID(val)).replace('-', '') == _format_uuid_string(val)
except (TypeError, ValueError, AttributeError): except (TypeError, ValueError, AttributeError):
return False return False
class TimeoutError(Exception):
pass
def timeout(seconds=10, error_message=os.strerror(errno.ETIME)):
def decorator(func):
def _handle_timeout(signum, frame):
raise TimeoutError(error_message)
def wrapper(*args, **kwargs):
signal.signal(signal.SIGALRM, _handle_timeout)
signal.alarm(seconds)
try:
result = func(*args, **kwargs)
finally:
signal.alarm(0)
return result
return wraps(func)(wrapper)
return decorator