Re-org the code to support "pip install"

Change-Id: Ifed51b19312b5373925f8685aa1eb8bd8770cf6d

README.rst

@@ -91,7 +91,8 @@ VMTP will display the results to stdout with the following data:
     |   |   - ICMP
     |   |   |  average, min, max and stddev round trip time in ms
 
-Detailed results can also be stored in a file in JSON format using the *--json* command line argument and/or stored directly into a MongoDB server. See :download:`here <_static/example.json>` for an example JSON file that is generated by VMTP.
+Detailed results can also be stored in a file in JSON format using the *--json* command line argument and/or stored directly into a MongoDB server. See `example.json <https://github.com/stackforge/vmtp/blob/master/doc/source/_static/example.json>`_ for an example JSON file that is generated by VMTP.
+
 The packaged python tool genchart.py can be used to generate from the JSON result files column charts in HTML format visible from any browser.
 
 Example of column chart generated by genchart.py:

cfg.existing.yaml

@@ -1,10 +0,0 @@
-#
-# Example of configuration where we froce the use of a specific external network and
-# use provider network (no floating IP)
-reuse_network_name : 'prov1'
-
-# Floating ip false is a provider network where we simply attach to it
-floating_ip : False
-
-# Floating ip is true by default:
-# attach to existing network, create a floating ip and attach instance to it

doc/source/conf.py

@@ -17,6 +17,8 @@ import os
 import re
 import sys
 
+from pbr import version as pbr_ver
+
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
@@ -58,11 +60,9 @@ copyright = u"%d, OpenStack Foundation" % datetime.datetime.now().year
 # built documents.
 #
 # The short X.Y version.
-vmtp_file = open("../../vmtp.py")
+version = pbr_ver.VersionInfo(project).version_string()
-raw_text = vmtp_file.read()
-version = re.search(r"__version__\s=\s'(\d+\.\d+\.\d+)'", raw_text).group(1)
 # The full version, including alpha/beta/rc tags.
-release = version
+release = pbr_ver.VersionInfo(project).version_string_with_vcs()
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

requirements.txt

@@ -2,7 +2,7 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 
-pbr>=0.6,!=0.7,<1.0
+pbr<2.0,>=1.3
 Babel>=1.3
 
 configure>=0.5

setup.cfg

@@ -8,22 +8,29 @@ author-email = openstack-dev@lists.openstack.org
 home-page = http://www.openstack.org/
 classifier =
     Environment :: OpenStack
+    Intended Audience :: Developers
     Intended Audience :: Information Technology
     Intended Audience :: System Administrators
     License :: OSI Approved :: Apache Software License
     Operating System :: POSIX :: Linux
+    Operating System :: MacOS
     Programming Language :: Python
     Programming Language :: Python :: 2
-    Programming Language :: Python :: 2.7
     Programming Language :: Python :: 2.6
-    Programming Language :: Python :: 3
+    Programming Language :: Python :: 2.7
-    Programming Language :: Python :: 3.3
-    Programming Language :: Python :: 3.4
 
 [files]
 packages =
     vmtp
 
+package_data =
+    vmtp =
+        tools
+
+[entry_points]
+console_scripts =
+    vmtp = vmtp.vmtp:main
+
 [build_sphinx]
 source-dir = doc/source
 build-dir = doc/build

test-requirements.txt

@@ -7,7 +7,7 @@ hacking>=0.9.2,<0.10
 coverage>=3.6
 discover
 python-subunit>=0.0.18
-sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3
+sphinx>=1.2.3
 testrepository>=0.0.18
 testscenarios>=0.4
 testtools>=0.9.36,!=1.2.0

tox.ini

@@ -1,6 +1,6 @@
 [tox]
 minversion = 1.6
-envlist = py33,py34,py26,py27,pypy,pep8
+envlist = py26,py27,pypy,pep8
 skipsdist = True
 
 [testenv]

vmtp/__init__.py

@@ -16,4 +16,4 @@ import pbr.version
 
 
 __version__ = pbr.version.VersionInfo(
-    'vmtp').version_string()
+    'vmtp').version_string_with_vcs()

vmtp/cfg.default.yaml

@@ -121,10 +121,6 @@ vm_name_client:   'TestClient'
 # name of the security group to create and use
 security_group_name: 'pns-security'
 
-# Location to the performance test tools.
-# If relative, is relative to the vmtp directory
-perf_tool_path: './tools'
-
 # ping variables
 ping_count: 2
 ping_pass_threshold: 80

vmtp/instance.py

@@ -18,12 +18,10 @@ import re
 import stat
 import subprocess
 
-
 import monitor
+from netaddr import IPAddress
 import sshutils
 
-
-from netaddr import IPAddress
 # a dictionary of sequence number indexed by a name prefix
 prefix_seq = {}
 

vmtp/iperf_tool.py

@@ -33,8 +33,8 @@ def get_bdw_kbps(bdw, bdw_unit):
 
 class IperfTool(PerfTool):
 
-    def __init__(self, instance, perf_tool_path):
+    def __init__(self, instance):
-        PerfTool.__init__(self, 'iperf', perf_tool_path, instance)
+        PerfTool.__init__(self, 'iperf', instance)
 
     def get_server_launch_cmd(self):
         '''Return the command to launch the server side.'''

vmtp/nuttcp_tool.py

@@ -20,8 +20,8 @@ import sshutils
 
 class NuttcpTool(PerfTool):
 
-    def __init__(self, instance, perf_tool_path):
+    def __init__(self, instance):
-        PerfTool.__init__(self, 'nuttcp-7.3.2', perf_tool_path, instance)
+        PerfTool.__init__(self, 'nuttcp-7.3.2', instance)
 
     def get_server_launch_cmd(self):
         '''Return the commands to launch the server side.'''

vmtp/perf_instance.py

@@ -27,7 +27,7 @@ class PerfInstance(Instance):
         else:
             self.ping = None
         if config.tp_tool:
-            self.tp_tool = config.tp_tool(self, config.perf_tool_path)
+            self.tp_tool = config.tp_tool(self)
         else:
             self.tp_tool = None
         # Override the config drive option to save in instance

vmtp/perf_tool.py

@@ -14,35 +14,31 @@
 #
 
 import abc
-import os
 import re
 
+from pkg_resources import resource_filename
+
 # where to copy the tool on the target, must end with slash
 SCP_DEST_DIR = '/tmp/'
 
-
-
 #
 # A base class for all tools that can be associated to an instance
 #
 class PerfTool(object):
     __metaclass__ = abc.ABCMeta
 
-    def __init__(self, name, perf_tool_path, instance):
+    def __init__(self, name, instance):
         self.name = name
         self.instance = instance
         self.dest_path = SCP_DEST_DIR + name
         self.pid = None
-        self.perf_tool_path = perf_tool_path
 
     # install the tool to the instance
     # returns False if fail, True if success
     def install(self):
-        if self.perf_tool_path:
+        self.instance.display('Installing %s...' % (self.name))
-            local_path = os.path.join(self.perf_tool_path, self.name)
+        local_path = resource_filename(__name__, 'tools/%s' % (self.name))
-            return self.instance.scp(self.name, local_path, self.dest_path)
+        return self.instance.scp(self.name, local_path, self.dest_path)
-        # no install needed
-        return True
 
     @abc.abstractmethod
     def get_server_launch_cmd(self):
@@ -225,7 +221,7 @@ class PingTool(PerfTool):
     '''
 
     def __init__(self, instance):
-        PerfTool.__init__(self, 'ping', None, instance)
+        PerfTool.__init__(self, 'ping', instance)
 
     def run_client(self, target_ip, ping_count=5):
         '''Perform the ping operation

vmtp/pnsdb_summary.py

@@ -20,7 +20,6 @@ import re
 import sys
 
 import pns_mongo
-
 import tabulate
 
 ###########################################

vmtp/vmtp.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python
 # Copyright 2014 Cisco Systems, Inc.  All rights reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -24,66 +25,32 @@ import re
 import sys
 import traceback
 
+from __init__ import __version__
 import compute
-import credentials
-import iperf_tool
-import network
-import nuttcp_tool
-import pns_mongo
-import sshutils
-
 import configure
+import credentials
 from glanceclient.v2 import client as glanceclient
+import iperf_tool
 from keystoneclient.v2_0 import client as keystoneclient
+import network
 from neutronclient.v2_0 import client as neutronclient
 from novaclient.client import Client
 from novaclient.exceptions import ClientException
-from prettytable import PrettyTable
+import nuttcp_tool
-
-__version__ = '2.1.2'
-
 from perf_instance import PerfInstance as PerfInstance
+from pkg_resources import resource_string
+import pns_mongo
+from prettytable import PrettyTable
+import sshutils
 
-def get_vmtp_absolute_path_for_file(file_name):
+flow_num = 0
-    '''
-    Return the filename in absolute path for any file
-    passed as relative path to the vmtp directory
-    '''
-    if os.path.isabs(__file__):
-        abs_file_path = os.path.join(__file__.split("vmtp.py")[0],
-                                     file_name)
-    else:
-        abs_file = os.path.abspath(__file__)
-        abs_file_path = os.path.join(abs_file.split("vmtp.py")[0],
-                                     file_name)
-
-    return abs_file_path
-
-
-def normalize_paths(cfg):
-    '''
-    Normalize the various paths to config files, tools, ssh priv and pub key
-    files.
-    If a relative path is entered:
-    - the key pair file names are relative to the current directory
-    - the perftool path is relative to vmtp itself
-    '''
-    if cfg.public_key_file:
-        cfg.public_key_file = os.path.abspath(os.path.expanduser(cfg.public_key_file))
-    if cfg.private_key_file:
-        cfg.private_key_file = os.path.expanduser(os.path.expanduser(cfg.private_key_file))
-    if cfg.perf_tool_path:
-        cfg.perf_tool_path = get_vmtp_absolute_path_for_file(cfg.perf_tool_path)
-
 class FlowPrinter(object):
-
+    @staticmethod
-    def __init__(self):
+    def print_desc(desc):
-        self.flow_num = 0
+        global flow_num
-
+        flow_num = flow_num + 1
-    def print_desc(self, desc):
-        self.flow_num += 1
         print "=" * 60
-        print('Flow %d: %s' % (self.flow_num, desc))
+        print('Flow %d: %s' % (flow_num, desc))
 
 class ResultsCollector(object):
 
@@ -154,7 +121,7 @@ class VmtpException(Exception):
     pass
 
 class VmtpTest(object):
-    def __init__(self):
+    def __init__(self, config, cred):
         '''
             1. Authenticate nova and neutron with keystone
             2. Create new client objects for neutron and nova
@@ -176,12 +143,16 @@ class VmtpTest(object):
         self.sec_group = None
         self.image_instance = None
         self.flavor_type = None
+        self.instance_access = None
+        self.rescol = ResultsCollector()
+        self.config = config
+        self.cred = cred
 
     # Create an instance on a particular availability zone
     def create_instance(self, inst, az, int_net):
         self.assert_true(inst.create(self.image_instance,
                                      self.flavor_type,
-                                     instance_access,
+                                     self.instance_access,
                                      int_net,
                                      az,
                                      int_net['name'],
@@ -192,49 +163,78 @@ class VmtpTest(object):
             raise VmtpException('Assert failure')
 
     def setup(self):
+        # This is a template host access that will be used for all instances
+        # (the only specific field specific to each instance is the host IP)
+        # For test VM access, we never use password and always need a key pair
+        self.instance_access = sshutils.SSHAccess()
+        self.instance_access.username = self.config.ssh_vm_username
+        # if the configuration does not have a
+        # key pair specified, we check if the user has a personal key pair
+        # if no key pair is configured or usable, a temporary key pair will be created
+        if self.config.public_key_file and self.config.private_key_file:
+            self.instance_access.public_key_file = self.config.public_key_file
+            self.instance_access.private_key_file = self.config.private_key_file
+        else:
+            pub_key = os.path.expanduser('~/.ssh/id_rsa.pub')
+            priv_key = os.path.expanduser('~/.ssh/id_rsa')
+            if os.path.isfile(pub_key) and os.path.isfile(priv_key):
+                self.instance_access.public_key_file = pub_key
+                self.instance_access.private_key_file = priv_key
+            else:
+                print('Error: Default keypair ~/.ssh/id_rsa[.pub] is not existed. Please '
+                      'either create one in your home directory, or specify your keypair '
+                      'information in the config file before running VMTP.')
+                sys.exit(1)
+
+        if self.config.debug and self.instance_access.public_key_file:
+            print('VM public key:  ' + self.instance_access.public_key_file)
+            print('VM private key: ' + self.instance_access.private_key_file)
+
         # If we need to reuse existing vms just return without setup
-        if not config.reuse_existing_vm:
+        if not self.config.reuse_existing_vm:
-            creds = cred.get_credentials()
+            creds = self.cred.get_credentials()
-            creds_nova = cred.get_nova_credentials_v2()
+            creds_nova = self.cred.get_nova_credentials_v2()
             # Create the nova and neutron instances
             nova_client = Client(**creds_nova)
             neutron = neutronclient.Client(**creds)
 
-            self.comp = compute.Compute(nova_client, config)
+            self.comp = compute.Compute(nova_client, self.config)
 
             # Add the appropriate public key to openstack
-            self.comp.init_key_pair(config.public_key_name, instance_access)
+            self.comp.init_key_pair(self.config.public_key_name, self.instance_access)
 
-            self.image_instance = self.comp.find_image(config.image_name)
+            self.image_instance = self.comp.find_image(self.config.image_name)
             if self.image_instance is None:
-                if config.vm_image_url != "":
+                if self.config.vm_image_url != "":
                     print '%s: image for VM not found, uploading it ...' \
-                        % (config.image_name)
+                        % (self.config.image_name)
                     keystone = keystoneclient.Client(**creds)
                     glance_endpoint = keystone.service_catalog.url_for(
                         service_type='image', endpoint_type='publicURL')
                     glance_client = glanceclient.Client(
                         glance_endpoint, token=keystone.auth_token)
                     self.comp.upload_image_via_url(
-                        creds, glance_client, config.image_name, config.vm_image_url)
+                        creds, glance_client,
-                    self.image_instance = self.comp.find_image(config.image_name)
+                        self.config.image_name,
+                        self.config.vm_image_url)
+                    self.image_instance = self.comp.find_image(self.config.image_name)
                 else:
                     # Exit the pogram
                     print '%s: image to launch VM not found. ABORTING.' \
-                        % (config.image_name)
+                        % (self.config.image_name)
                     sys.exit(1)
 
             self.assert_true(self.image_instance)
-            print 'Found image %s to launch VM, will continue' % (config.image_name)
+            print 'Found image %s to launch VM, will continue' % (self.config.image_name)
-            self.flavor_type = self.comp.find_flavor(config.flavor_type)
+            self.flavor_type = self.comp.find_flavor(self.config.flavor_type)
-            self.net = network.Network(neutron, config)
+            self.net = network.Network(neutron, self.config)
 
-            rescol.add_property('l2agent_type', self.net.l2agent_type)
+            self.rescol.add_property('l2agent_type', self.net.l2agent_type)
             print "OpenStack agent: " + self.net.l2agent_type
             try:
                 network_type = self.net.vm_int_net[0]['provider:network_type']
                 print "OpenStack network type: " + network_type
-                rescol.add_property('encapsulation', network_type)
+                self.rescol.add_property('encapsulation', network_type)
             except KeyError as exp:
                 network_type = 'Unknown'
                 print "Provider network type not found: ", str(exp)
@@ -243,25 +243,25 @@ class VmtpTest(object):
         self.sec_group = self.comp.security_group_create()
         if not self.sec_group:
             raise VmtpException("Security group creation failed")
-        if config.reuse_existing_vm:
+        if self.config.reuse_existing_vm:
-            self.server.internal_ip = config.vm_server_internal_ip
+            self.server.internal_ip = self.config.vm_server_internal_ip
-            self.client.internal_ip = config.vm_client_internal_ip
+            self.client.internal_ip = self.config.vm_client_internal_ip
-            if config.vm_server_external_ip:
+            if self.config.vm_server_external_ip:
-                self.server.ssh_access.host = config.vm_server_external_ip
+                self.server.ssh_access.host = self.config.vm_server_external_ip
             else:
-                self.server.ssh_access.host = config.vm_server_internal_ip
+                self.server.ssh_access.host = self.config.vm_server_internal_ip
-            if config.vm_client_external_ip:
+            if self.config.vm_client_external_ip:
-                self.client.ssh_access.host = config.vm_client_external_ip
+                self.client.ssh_access.host = self.config.vm_client_external_ip
             else:
-                self.client.ssh_access.host = config.vm_client_internal_ip
+                self.client.ssh_access.host = self.config.vm_client_internal_ip
             return
 
         # this is the standard way of running the test
         # NICs to be used for the VM
-        if config.reuse_network_name:
+        if self.config.reuse_network_name:
             # VM needs to connect to existing management and new data network
             # Reset the management network name
-            config.internal_network_name[0] = config.reuse_network_name
+            self.config.internal_network_name[0] = self.config.reuse_network_name
         else:
             # Make sure we have an external network and an external router
             self.assert_true(self.net.ext_net)
@@ -279,14 +279,14 @@ class VmtpTest(object):
         server_az = avail_list[0]
         if len(avail_list) > 1:
             # 2 hosts are known
-            if config.inter_node_only:
+            if self.config.inter_node_only:
                 # in this case we do not want the client to run on the same host
                 # as the server
                 avail_list.pop(0)
         self.client_az_list = avail_list
 
-        self.server = PerfInstance(config.vm_name_server,
+        self.server = PerfInstance(self.config.vm_name_server,
-                                   config,
+                                   self.config,
                                    self.comp,
                                    self.net,
                                    server=True)
@@ -296,20 +296,20 @@ class VmtpTest(object):
 
     # Test throughput for the case of the external host
     def ext_host_tp_test(self):
-        client = PerfInstance('Host-' + config.ext_host.host + '-Client', config)
+        client = PerfInstance('Host-' + self.config.ext_host.host + '-Client', self.config)
-        if not client.setup_ssh(config.ext_host):
+        if not client.setup_ssh(self.config.ext_host):
             client.display('SSH to ext host failed, check IP or make sure public key is configured')
         else:
             client.buginf('SSH connected')
             client.create()
-            fpr.print_desc('External-VM (upload/download)')
+            FlowPrinter.print_desc('External-VM (upload/download)')
             res = client.run_client('External-VM',
                                     self.server.ssh_access.host,
                                     self.server,
-                                    bandwidth=config.vm_bandwidth,
+                                    bandwidth=self.config.vm_bandwidth,
                                     bidirectional=True)
             if res:
-                rescol.add_flow_result(res)
+                self.rescol.add_flow_result(res)
             client.dispose()
 
     def add_location(self, label):
@@ -324,7 +324,8 @@ class VmtpTest(object):
         return label
 
     def create_flow_client(self, client_az, int_net):
-        self.client = PerfInstance(config.vm_name_client, config,
+        self.client = PerfInstance(self.config.vm_name_client,
+                                   self.config,
                                    self.comp,
                                    self.net)
         self.client.display('Creating client VM...')
@@ -332,23 +333,23 @@ class VmtpTest(object):
 
     def measure_flow(self, label, target_ip):
         label = self.add_location(label)
-        fpr.print_desc(label)
+        FlowPrinter.print_desc(label)
 
         # results for this flow as a dict
         perf_output = self.client.run_client(label, target_ip,
                                              self.server,
-                                             bandwidth=config.vm_bandwidth,
+                                             bandwidth=self.config.vm_bandwidth,
                                              az_to=self.server.az)
-        if opts.stop_on_error:
+        if self.config.stop_on_error:
             # check if there is any error in the results
             results_list = perf_output['results']
             for res_dict in results_list:
                 if 'error' in res_dict:
                     print('Stopping execution on error, cleanup all VMs/networks manually')
-                    rescol.pprint(perf_output)
+                    self.rescol.pprint(perf_output)
                     sys.exit(2)
 
-        rescol.add_flow_result(perf_output)
+        self.rescol.add_flow_result(perf_output)
 
     def measure_vm_flows(self):
         # scenarios need to be tested for both inter and intra node
@@ -363,13 +364,13 @@ class VmtpTest(object):
                               self.server.internal_ip)
             self.client.dispose()
             self.client = None
-            if not config.reuse_network_name:
+            if not self.config.reuse_network_name:
                 # Different network
                 self.create_flow_client(client_az, self.net.vm_int_net[1])
 
                 self.measure_flow("VM to VM different network fixed IP",
                                   self.server.internal_ip)
-                if not config.ipv6_mode:
+                if not self.config.ipv6_mode:
                     self.measure_flow("VM to VM different network floating IP",
                                       self.server.ssh_access.host)
 
@@ -377,7 +378,7 @@ class VmtpTest(object):
                 self.client = None
 
         # If external network is specified run that case
-        if config.ext_host:
+        if self.config.ext_host:
             self.ext_host_tp_test()
 
     def teardown(self):
@@ -389,11 +390,11 @@ class VmtpTest(object):
             self.server.dispose()
         if self.client:
             self.client.dispose()
-        if not config.reuse_existing_vm and self.net:
+        if not self.config.reuse_existing_vm and self.net:
             self.net.dispose()
         # Remove the public key
         if self.comp:
-            self.comp.remove_public_key(config.public_key_name)
+            self.comp.remove_public_key(self.config.public_key_name)
         # Finally remove the security group
         try:
             if self.comp:
@@ -415,14 +416,15 @@ class VmtpTest(object):
             traceback.print_exc()
             error_flag = True
 
-        if opts.stop_on_error and error_flag:
+        if self.config.stop_on_error and error_flag:
             print('Stopping execution on error, cleanup all VMs/networks manually')
             sys.exit(2)
         else:
             self.teardown()
 
-def test_native_tp(nhosts, ifname):
+def test_native_tp(nhosts, ifname, config):
-    fpr.print_desc('Native Host to Host throughput')
+    FlowPrinter.print_desc('Native Host to Host throughput')
+    result_list = None
     server_host = nhosts[0]
     server = PerfInstance('Host-' + server_host.host + '-Server', config, server=True)
 
@@ -464,57 +466,17 @@ def test_native_tp(nhosts, ifname):
                                             server_ip,
                                             server,
                                             bandwidth=config.vm_bandwidth)
-                    rescol.add_flow_result(res)
+                    result_list.append(res)
                 client.dispose()
     server.dispose()
 
-def _get_ssh_access(opt_name, opt_value):
+    return result_list
-    '''Allocate a HostSshAccess instance to the option value
-    Check that a password is provided or the key pair in the config file
-    is valid.
-    If invalid exit with proper error message
-    '''
-    if not opt_value:
-        return None
 
-    host_access = sshutils.SSHAccess(opt_value)
+def get_controller_info(ssh_access, net, res_col, retry_count):
-    host_access.private_key_file = config.private_key_file
-    host_access.public_key_file = config.public_key_file
-    if host_access.error:
-        print'Error for --' + (opt_name + ':' + host_access.error)
-        sys.exit(2)
-    return host_access
-
-def _merge_config(cfg_file, source_config, required=False):
-    '''
-    returns the merged config or exits if the file does not exist and is required
-    '''
-    dest_config = source_config
-
-    fullname = os.path.expanduser(cfg_file)
-    if os.path.isfile(fullname):
-        print('Loading ' + fullname + '...')
-        try:
-            alt_config = configure.Configuration.from_file(fullname).configure()
-            dest_config = source_config.merge(alt_config)
-
-        except configure.ConfigurationError:
-            # this is in most cases when the config file passed is empty
-            # configure.ConfigurationError: unconfigured
-            # in case of syntax error, another exception is thrown:
-            # TypeError: string indices must be integers, not str
-            pass
-    elif required:
-        print('Error: configration file %s does not exist' % (fullname))
-        sys.exit(1)
-    return dest_config
-
-def get_controller_info(ssh_access, net, res_col):
     if not ssh_access:
         return
     print 'Fetching OpenStack deployment details...'
-    sshcon = sshutils.SSH(ssh_access,
+    sshcon = sshutils.SSH(ssh_access, connect_retry_count=retry_count)
-                          connect_retry_count=config.ssh_retry_count)
     if sshcon is None:
         print 'ERROR: Cannot connect to the controller node'
         return
@@ -648,14 +610,63 @@ def print_report(results):
     print "Skipped Scenarios: %d" % (cnt_skipped)
     print ptable
 
+def normalize_paths(cfg):
+    '''
+    Normalize the various paths to config files, tools, ssh priv and pub key
+    files.
+    If a relative path is entered:
+    - the key pair file names are relative to the current directory
+    - the perftool path is relative to vmtp itself
+    '''
+    if cfg.public_key_file:
+        cfg.public_key_file = os.path.abspath(os.path.expanduser(cfg.public_key_file))
+    if cfg.private_key_file:
+        cfg.private_key_file = os.path.expanduser(os.path.expanduser(cfg.private_key_file))
 
-if __name__ == '__main__':
+def main():
+    def _get_ssh_access(opt_name, opt_value):
+        '''Allocate a HostSshAccess instance to the option value
+        Check that a password is provided or the key pair in the config file
+        is valid.
+        If invalid exit with proper error message
+        '''
+        if not opt_value:
+            return None
+
+        host_access = sshutils.SSHAccess(opt_value)
+        host_access.private_key_file = config.private_key_file
+        host_access.public_key_file = config.public_key_file
+        if host_access.error:
+            print'Error for --' + (opt_name + ':' + host_access.error)
+            sys.exit(2)
+        return host_access
+
+    def _merge_config(cfg_file, source_config, required=False):
+        '''
+        returns the merged config or exits if the file does not exist and is required
+        '''
+        dest_config = source_config
+
+        fullname = os.path.expanduser(cfg_file)
+        if os.path.isfile(fullname):
+            print('Loading ' + fullname + '...')
+            try:
+                alt_config = configure.Configuration.from_file(fullname).configure()
+                dest_config = source_config.merge(alt_config)
+
+            except configure.ConfigurationError:
+                # this is in most cases when the config file passed is empty
+                # configure.ConfigurationError: unconfigured
+                # in case of syntax error, another exception is thrown:
+                # TypeError: string indices must be integers, not str
+                pass
+        elif required:
+            print('Error: configration file %s does not exist' % (fullname))
+            sys.exit(1)
+        return dest_config
 
-    fpr = FlowPrinter()
-    rescol = ResultsCollector()
     logging.basicConfig()
-
+    parser = argparse.ArgumentParser(description='OpenStack VM Throughput ' + __version__)
-    parser = argparse.ArgumentParser(description='OpenStack VM Throughput V' + __version__)
 
     parser.add_argument('-c', '--config', dest='config',
                         action='store',
@@ -791,25 +802,25 @@ if __name__ == '__main__':
 
     (opts, args) = parser.parse_known_args()
 
-    default_cfg_file = get_vmtp_absolute_path_for_file("cfg.default.yaml")
+    default_cfg_file = resource_string(__name__, "cfg.default.yaml")
 
     # read the default configuration file and possibly an override config file
     # the precedence order is as follows:
     # $HOME/.vmtp.yaml if exists
     # -c <file> from command line if provided
     # cfg.default.yaml
-    config = configure.Configuration.from_file(default_cfg_file).configure()
+    config = configure.Configuration.from_string(default_cfg_file).configure()
     config = _merge_config('~/.vmtp.yaml', config)
 
     if opts.config:
         config = _merge_config(opts.config, config, required=True)
 
     if opts.version:
-        print('Version ' + __version__)
+        print(__version__)
         sys.exit(0)
 
-    # debug flag
     config.debug = opts.debug
+    config.stop_on_error = opts.stop_on_error
     config.inter_node_only = opts.inter_node_only
 
     if config.public_key_file and not os.path.isfile(config.public_key_file):
@@ -824,7 +835,6 @@ if __name__ == '__main__':
         print('Invalid vnic-type: ' + opts.vnic_type)
         sys.exit(1)
     config.vnic_type = opts.vnic_type
-
     config.hypervisors = opts.hypervisors
 
     # time to run each perf test in seconds
@@ -841,46 +851,12 @@ if __name__ == '__main__':
     # Initialize the external host access
     config.ext_host = _get_ssh_access('external-host', opts.ext_host)
 
-    # This is a template host access that will be used for all instances
-    # (the only specific field specific to each instance is the host IP)
-    # For test VM access, we never use password and always need a key pair
-    instance_access = sshutils.SSHAccess()
-    instance_access.username = config.ssh_vm_username
-    # if the configuration does not have a
-    # key pair specified, we check if the user has a personal key pair
-    # if no key pair is configured or usable, a temporary key pair will be created
-    if config.public_key_file and config.private_key_file:
-        instance_access.public_key_file = config.public_key_file
-        instance_access.private_key_file = config.private_key_file
-    else:
-        pub_key = os.path.expanduser('~/.ssh/id_rsa.pub')
-        priv_key = os.path.expanduser('~/.ssh/id_rsa')
-        if os.path.isfile(pub_key) and os.path.isfile(priv_key):
-            instance_access.public_key_file = pub_key
-            instance_access.private_key_file = priv_key
-        else:
-            print('Error: Default keypair ~/.ssh/id_rsa[.pub] is not existed. Please '
-                  'either create one in your home directory, or specify your keypair '
-                  'information in the config file before running VMTP.')
-            sys.exit(1)
-
-    if opts.debug and instance_access.public_key_file:
-        print('VM public key:  ' + instance_access.public_key_file)
-        print('VM private key: ' + instance_access.private_key_file)
-
-
     ###################################################
     # VM Image URL
     ###################################################
     if opts.vm_image_url:
         config.vm_image_url = opts.vm_image_url
 
-    ###################################################
-    # Test Description
-    ###################################################
-    if opts.test_description:
-        rescol.add_property('test_description', opts.test_description)
-
     ###################################################
     # MongoDB Server connection info.
     ###################################################
@@ -986,37 +962,51 @@ if __name__ == '__main__':
                     if_name = last_arg
                 host = host[:last_column_index]
             native_hosts.append(_get_ssh_access('host', host))
-        test_native_tp(native_hosts, if_name)
+        native_tp_results = test_native_tp(native_hosts, if_name, config)
-
+    else:
-    cred = credentials.Credentials(opts.rc, opts.pwd, opts.no_env)
+        native_tp_results = []
 
     # replace all command line arguments (after the prog name) with
     # those args that have not been parsed by this parser so that the
     # unit test parser is not bothered by local arguments
     sys.argv[1:] = args
     vmtp_net = None
+    vmtp_instance = None
+
+    cred = credentials.Credentials(opts.rc, opts.pwd, opts.no_env)
     if cred.rc_auth_url:
-        if opts.debug:
+        if config.debug:
             print 'Using ' + cred.rc_auth_url
-        rescol.add_property('auth_url', cred.rc_auth_url)
+        vmtp_instance = VmtpTest(config, cred)
-        vmtp = VmtpTest()
+        for item in native_tp_results:
-        vmtp.run()
+            vmtp_instance.rescol.add_flow_result(item)
-        vmtp_net = vmtp.net
+        vmtp_instance.run()
+        vmtp_net = vmtp_instance.net
 
     # Retrieve controller information if requested
     # controller node ssh access to collect metadata for the run.
     ctrl_host_access = _get_ssh_access('controller-node', opts.controller_node)
-    get_controller_info(ctrl_host_access, vmtp_net, rescol)
+    get_controller_info(ctrl_host_access,
+                        vmtp_net,
+                        vmtp_instance.rescol,
+                        config.ssh_retry_count)
 
-    print_report(rescol.results)
+    print_report(vmtp_instance.rescol.results)
 
     # If saving the results to JSON or MongoDB, get additional details:
     if config.json_file or config.vmtp_mongod_ip:
-        rescol.mask_credentials()
+        vmtp_instance.rescol.mask_credentials()
-        rescol.generate_runid()
+        vmtp_instance.rescol.generate_runid()
 
     if config.json_file:
-        rescol.save(config)
+        # Test Description
+        if opts.test_description:
+            vmtp_instance.rescol.add_property('test_description', opts.test_description)
+        vmtp_instance.rescol.add_property('auth_url', cred.rc_auth_url)
+        vmtp_instance.rescol.save(config)
 
     if config.vmtp_mongod_ip:
-        rescol.save_to_db(config)
+        vmtp_instance.rescol.save_to_db(config)
+
+if __name__ == '__main__':
+    main()