Refactor iperf CLI

This patch contains these features:
1. add new param to iperf CLI
2. add "validate_ip" in agent
3. rename strutils.py into utils.py
4. move some common functions into utils.py
5. modify doc and configuration file

Change-Id: I993041b3d91bd42e9d732f07383d6d92b2f5d3d3

doc/source/index.rst

@@ -55,13 +55,27 @@ Configuration File
 On start the client will read a configuration file. By default the configuration file is located at /etc/steth/steth.conf.
 Here is an example about the configuration file: ::
 
-    [DEFAULT]
+  # (ListOpt) Order list of networks prefix.
-    # Prefix of managed network in every agent. End of '.'
+  # The first item is treated as a list.
-    # Example: "10.0.4."
+  # If multiple networks are used, we can be specified as s list.
-    manage_network_prefix=127.0.0.
+  # Specify the prefix of the networks to be used.
-    # Network nodes info. Just need sequence number.
+  # The ending '.' -- specifier indicates the network range to be used.
+  # Example: "10.0.4.,192.168.10."
+  networks_prefix=127.0.0.,192.168.20.,1.1.1.
+
+  # (ListOpt) This is the identifier of the nodes in group of network nodes.
   # Example: 64, 65, 66
   network_agents_info=64,65,66
-    # Compute nodes info. Just need sequence number.
+
+  # (ListOpt) This is the identifier of the nodes in group of compute nodes.
   # Example: 67, 68
   compute_agents_info=67,68
+
+  # (StrOpt) Prefix to be used in naming every node. By default, this value
+  # is "server". We combine "node_name_prefix" with
+  # "network_agents_info", "compute_agents_info" to
+  # define nodes. Such as "server-64", "server-68" and so on.
+  # In every region, we give every node a specific name.
+  # Ensure that DNS can be resolved correctly.
+  # these names when doing iperf.
+  node_name_prefix=server-

etc/steth.conf

@@ -1,12 +1,24 @@
 [DEFAULT]
-# Prefix of managed network. End of '.'
-# Example: "10.0.4."
-manage_network_prefix=127.0.0.
 
-# Network nodes info. Just need sequence number.
+# (ListOpt) Order list of networks prefix.
+# We treat first item as managed network.
+# We may have multi networks in one node, so this value should be a list.
+# Prefix of network in every agent. End of '.'
+# Example: "10.0.4.,192.168.10."
+networks_prefix=127.0.0.,192.168.20.,1.1.1.
+
+# (ListOpt) Network nodes info. Just need sequence number.
 # Example: 64, 65, 66
 network_agents_info=64,65,66
 
-# Compute nodes info. Just need sequence number.
+# (ListOpt) Compute nodes info. Just need sequence number.
 # Example: 67, 68
 compute_agents_info=67,68
+
+# (StrOpt) Name prefix of every node. By default, this value
+# is "server". We combine "node_name_prefix" with
+# "network_agents_info", "compute_agents_info" to
+# define nodes. Such as "server-64", "server-68" and so on.
+# In every region, we give every node a specific name.
+# Ensure that DNS can resolve the nodes.
+node_name_prefix=server-

steth/agent/api.py

@@ -47,9 +47,27 @@ class AgentApi(object):
         return agent_utils.make_response(code=stdcode,
                                          message=message)
 
+    def validate_ip(self, ip=None):
+        """Validate if IP exists on agent
+
+        ip addr show to ip
+        """
+        if not ip:
+            msg = "Please validate the IP address"
+            return agent_utils.make_response(code=1,
+                                             message=msg)
+        LOG.info("RPC: validate_ip for %s" % ip)
+        cmd = ['ip', 'addr', 'show', 'to', ip]
+        stdcode, stdout = agent_utils.execute(cmd, root=True)
+        if stdout:
+            return agent_utils.make_response(code=0)
+        msg = "IP: %s doesn't exist!" % ip
+        return agent_utils.make_response(code=1,
+                                         message=msg)
+
     def ping(self, ips, boardcast=False,
              count=2, timeout=2, interface=None):
-        """Ping host or broadcast.
+        """Ping host or perform broadcast ping.
 
         ping host -c 2 -W 2
         """

steth/stethclient/agent_api.py

@@ -15,62 +15,16 @@
 #    under the License.
 
 import logging
-import jsonrpclib
 import sys
 
 from cliff.command import Command
 from cliff.lister import Lister
+from steth.stethclient.utils import Logger
+from steth.stethclient.utils import setup_server
 
-LISTEN_PORT = 9698
 SETUP_LINK_IP_PRE = "192.168.100."
 
 
-class Logger():
-    HEADER = '\033[95m'
-    OKBLUE = '\033[94m'
-    OKGREEN = '\033[92m'
-    WARNING = '\033[93m'
-    FAIL = '\033[91m'
-    ENDC = '\033[0m'
-
-    @staticmethod
-    def log_normal(info):
-        print Logger.OKBLUE + info + Logger.ENDC
-
-    @staticmethod
-    def log_high(info):
-        print Logger.OKGREEN + info + Logger.ENDC
-
-    @staticmethod
-    def log_fail(info):
-        print Logger.FAIL + info + Logger.ENDC
-
-
-try:
-    from steth.stethclient.constants import AGENT_INFOS
-except:
-    AGENT_INFOS = {
-        'agent-64': "127.0.0.1",
-        'agent-65': "127.0.0.1",
-    }
-    Logger.log_fail("Import steth configure file fail. Use fake data!")
-
-
-def setup_server(agent):
-    log = logging.getLogger(__name__)
-    if agent in AGENT_INFOS:
-        log.debug('get agent:%s ip_address:%s' % (
-            agent, AGENT_INFOS[agent]))
-    else:
-        log.error('Agent %s not configured. Please check it.' % (agent))
-        sys.exit()
-    log.debug('Begin create connection with http://%s:9698.' % (agent))
-    server = jsonrpclib.Server('http://%s:%s' %
-                               (AGENT_INFOS[agent], LISTEN_PORT))
-    log.debug('Create connection with %s success.' % (agent))
-    return server
-
-
 class TearDownLink(Command):
     "Delete a link"
 

steth/stethclient/constants.py

@@ -22,14 +22,29 @@ OPTS = [
                 help="Mappings of compute agents and steth listened IP."),
     cfg.StrOpt('managed_network_prefix', default='127.0.0.',
                help="Managed network prefix."),
+    cfg.ListOpt('networks_prefix', default=['127.0.0.', '192.168.10.'],
+                help="Networks prefix."),
+    cfg.StrOpt('node_name_prefix', default='server-',
+               help="Prefix of every node."),
 ]
 
 cfg.CONF.register_opts(OPTS)
 cfg.CONF([], project='steth',
          default_config_files=['/etc/steth/steth.conf'])
 
-AGENT_INFOS = {}
 all_agents = cfg.CONF.network_agents_info + cfg.CONF.compute_agents_info
+
+# We use STETH_AGENT_INFOS to create connection to every node
+STETH_AGENT_INFOS = {}
+
+# We use ALL_AGENT_INFOS to process iperf
+ALL_AGENT_INFOS = {}
 for agent in all_agents:
-    item = {'agent-' + agent: cfg.CONF.managed_network_prefix + agent}
+    l = []
-    AGENT_INFOS.update(item)
+    prefix = cfg.CONF.networks_prefix[0]
+    item = {cfg.CONF.node_name_prefix + agent: prefix + agent}
+    STETH_AGENT_INFOS.update(item)
+    for prefix in cfg.CONF.networks_prefix[1:]:
+        l.append(prefix + agent)
+        item = {cfg.CONF.node_name_prefix + agent: l}
+        ALL_AGENT_INFOS.update(item)

steth/stethclient/drivers/iperf_api.py

@@ -15,58 +15,13 @@
 #    under the License.
 
 import logging
-import jsonrpclib
 import socket
 import sys
 
 from cliff.lister import Lister
-
+from steth.stethclient.utils import Logger
-LISTEN_PORT = 9698
+from steth.stethclient.utils import setup_server
-
+from steth.stethclient import utils
-
-class Logger():
-    HEADER = '\033[95m'
-    OKBLUE = '\033[94m'
-    OKGREEN = '\033[92m'
-    WARNING = '\033[93m'
-    FAIL = '\033[91m'
-    ENDC = '\033[0m'
-
-    @staticmethod
-    def log_normal(info):
-        print Logger.OKBLUE + info + Logger.ENDC
-
-    @staticmethod
-    def log_high(info):
-        print Logger.OKGREEN + info + Logger.ENDC
-
-    @staticmethod
-    def log_fail(info):
-        print Logger.FAIL + info + Logger.ENDC
-
-try:
-    from steth.stethclient.constants import AGENT_INFOS
-except:
-    AGENT_INFOS = {
-        'agent-64': "127.0.0.1",
-        'agent-65': "127.0.0.1",
-    }
-    Logger.log_fail("Import steth configure file fail. Use fake data!")
-
-
-def setup_server(agent):
-    log = logging.getLogger(__name__)
-    if agent in AGENT_INFOS:
-        log.debug('get agent:%s ip_address:%s' % (
-            agent, AGENT_INFOS[agent]))
-    else:
-        log.error('Agent %s not configured. Please check it.' % (agent))
-        sys.exit()
-    log.debug('Begin create connection with http://%s:9698.' % (agent))
-    server = jsonrpclib.Server('http://%s:%s' %
-                               (AGENT_INFOS[agent], LISTEN_PORT))
-    log.debug('Create connection with %s success.' % (agent))
-    return server
 
 
 def get_ip_by_hostname(hostname):
@@ -83,6 +38,7 @@ class CheckIperf(Lister):
         parser = super(CheckIperf, self).get_parser(prog_name)
         parser.add_argument('server_agent', default='bad')
         parser.add_argument('client_agent', default='bad')
+        parser.add_argument('iperf_server_ip', default='bad')
         parser.add_argument('--server_protocol', nargs='?', default='TCP')
         parser.add_argument('--server_port', nargs='?', default='5001')
         parser.add_argument('--client_protocol', nargs='?', default='TCP')
@@ -94,25 +50,38 @@ class CheckIperf(Lister):
 
     def take_action(self, parsed_args):
         self.log.debug('Get parsed_args: %s' % parsed_args)
+        # check iperf client ip if legal
+        if utils.is_ip(parsed_args.iperf_server_ip):
+            Logger.log_fail('IP address not valid')
+            sys.exit()
         server = setup_server(parsed_args.server_agent)
         client = setup_server(parsed_args.client_agent)
         iperf_server_pdid = None
+        # check iperf server ip exist
+        res = server.validate_ip(parsed_args.iperf_server_ip)
+        if res['code'] == 1:
+            Logger.log_fail(res['message'])
+            sys.exit()
         # setup iperf server
         res = server.setup_iperf_server(protocol=parsed_args.server_protocol,
                                         port=parsed_args.server_port)
         self.log.debug('Response is %s' % res)
-        if res['code'] == 1:
+        if res['code'] != 0:
             Logger.log_fail(res['message'])
             sys.exit()
         if res['code'] == 0:
             msg = (('Iperf server setup success and runs in '
                     'pid:%s') % (res['data']['pid']))
-            Logger.log_high(msg)
+            self.log.debug(msg)
             iperf_server_pdid = res['data']['pid']
         # setup iperf client
-        host = get_ip_by_hostname(parsed_args.server_agent)
+        #try:
+        #    host = get_ip_by_hostname(parsed_args.server_agent)
+        #except Exception as e:
+        #    self.log.info("We can not resolve this name: %s",
+        #        (parsed_args.server_agent))
         res = client.start_iperf_client(protocol=parsed_args.client_protocol,
-                                        host=host,
+                                        host=parsed_args.iperf_server_ip,
                                         timeout=parsed_args.client_timeout,
                                         parallel=parsed_args.client_parallel,
                                         bandwidth=parsed_args.client_bandwidth,
@@ -122,10 +91,9 @@ class CheckIperf(Lister):
         r = server.teardown_iperf_server(iperf_server_pdid)
         if r['code'] == 1:
             Logger.log_fail(r['message'])
-            sys.exit()
         if r['code'] == 0:
             msg = (('Iperf server delete success and '
                     'pid:%s') % (iperf_server_pdid))
-            Logger.log_high(msg)
+            self.log.debug(msg)
         return (('Field', 'Value'),
                 ((k, v) for k, v in res['data'].items()))

steth/stethclient/shell.py

@@ -22,7 +22,7 @@ from cliff import app
 from cliff import commandmanager
 from steth.stethclient import agent_api
 from steth.stethclient.drivers import iperf_api
-from steth.stethclient import strutils
+from steth.stethclient import utils
 
 
 VERSION = '0.1'
@@ -69,7 +69,7 @@ class StethShell(app.App):
 def main(argv=sys.argv[1:]):
     try:
         return StethShell(STETH_API_VERSION).run(
-            list(map(strutils.safe_decode, argv)))
+            map(utils.safe_decode, argv))
     except KeyboardInterrupt:
         print "... terminating neutron client"
         return 1

steth/stethclient/utils.py

@@ -13,7 +13,10 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import logging
+import jsonrpclib
 import six
+import socket
 import sys
 
 
@@ -53,3 +56,61 @@ def safe_decode(text, incoming=None, errors='strict'):
         # Also, UTF-8 is being used since it's an ASCII
         # extension.
         return text.decode('utf-8', errors)
+
+
+class Logger():
+    HEADER = '\033[95m'
+    OKBLUE = '\033[94m'
+    OKGREEN = '\033[92m'
+    WARNING = '\033[93m'
+    FAIL = '\033[91m'
+    ENDC = '\033[0m'
+
+    @staticmethod
+    def log_normal(info):
+        print Logger.OKBLUE + info + Logger.ENDC
+
+    @staticmethod
+    def log_high(info):
+        print Logger.OKGREEN + info + Logger.ENDC
+
+    @staticmethod
+    def log_fail(info):
+        print Logger.FAIL + info + Logger.ENDC
+
+LISTEN_PORT = 9698
+
+try:
+    from steth.stethclient.constants import STETH_AGENT_INFOS
+except:
+    STETH_AGENT_INFOS = {
+        'agent-64': "127.0.0.1",
+        'agent-65': "127.0.0.1",
+    }
+    Logger.log_fail("Import steth configure file fail. Use fake data!")
+
+
+def setup_server(agent):
+    log = logging.getLogger(__name__)
+    if agent in STETH_AGENT_INFOS:
+        log.debug('get agent:%s ip_address:%s' % (
+            agent, STETH_AGENT_INFOS[agent]))
+    else:
+        log.error('Agent %s not configured. Please check it.' % (agent))
+        sys.exit()
+    log.debug('Begin create connection with http://%s:%s.' % (agent,
+              LISTEN_PORT))
+    server = jsonrpclib.Server('http://%s:%s' %
+                               (STETH_AGENT_INFOS[agent], LISTEN_PORT))
+    log.debug('Create connection with %s success.' % (agent))
+    return server
+
+
+def is_ip(addr):
+    try:
+        socket.inet_aton(addr)
+        # legal
+        return 0
+    except socket.error:
+        # Not legal
+        return 1

steth/tests/unit/agent/test_api.py

@@ -83,3 +83,9 @@ class TestApi(unittest.TestCase):
         agent_utils.execute_wait = mock.Mock(return_value=(0, stdout, ''))
         self.agent_api.start_iperf_client(host='127.0.0.1')
         self.assertEqual(agent_utils.make_response.called, True)
+
+    def test_validate_ip(self):
+        stdout = ['', '']
+        agent_utils.execute = mock.Mock(return_value=(0, stdout))
+        self.agent_api.validate_ip('1.2.3.4')
+        self.assertEqual(agent_utils.make_response.called, True)

steth/tests/unit/stethclient/test_stethclient.py

@@ -74,6 +74,7 @@ class TestStethClientMethods(unittest.TestCase):
         self.assertEqual(self.server.check_ports_on_br.called, True)
 
     def test_check_iperf(self):
+        validate_ip_r = {'message': u'', 'code': 0, 'data': {}}
         iperf_server_r = {'message': '', 'code': 0, 'data': {'pid': 1234}}
         iperf_client_r = {
             'message': '',
@@ -88,11 +89,12 @@ class TestStethClientMethods(unittest.TestCase):
         teardown_iperf_r = {'message': '', 'code': 0, 'data': {}}
         self.server.setup_iperf_server = mock.Mock(return_value=iperf_server_r)
         self.server.start_iperf_client = mock.Mock(return_value=iperf_client_r)
+        self.server.validate_ip = mock.Mock(return_value=validate_ip_r)
         self.server.teardown_iperf_server = mock.Mock(
             return_value=teardown_iperf_r)
-        iperf_api.get_ip_by_hostname = mock.Mock(return_value='10.0.0.64')
+        #iperf_api.get_ip_by_hostname = mock.Mock(return_value='10.0.0.64')
-        shell.main(['check-iperf', 'agent-64', 'agent-64'])
+        shell.main(['check-iperf', 'agent-64', 'agent-64', '10.0.0.64'])
         self.assertEqual(self.server.setup_iperf_server.called, True)
-        self.assertEqual(iperf_api.get_ip_by_hostname.called, True)
         self.assertEqual(self.server.start_iperf_client.called, True)
+        self.assertEqual(self.server.validate_ip.called, True)
         self.assertEqual(self.server.teardown_iperf_server.called, True)