Allow availability zone auto-fill

Change-Id: I4fbe8089271e58220bcf3b2e41c95cabaabf41c1

README.rst

@@ -57,7 +57,6 @@ For running VMTP Docker Image
 
 Docker is installed. See `here <https://docs.docker.com/installation/#installation/>`_ for instructions.
 
-
 Sample Results Output
 ---------------------
 

cfg.default.yaml

@@ -2,7 +2,7 @@
 # VMTP default configuration file
 #
 # This configuration file is ALWAYS loaded by VMTP and should never be modified by users.
-# To specify your own user-specific property values, always define them in a separate config file
+# To specify your own property values, always define them in a separate config file
 # and pass that file to the script using -c or --config <file>
 # Property values in that config file will override the default values in the current file
 #
@@ -29,7 +29,10 @@ flavor_type: 'm1.small'
 # If the zone selected contains more than 1 compute node, the script
 # will determine inter-node and intra-node throughput. If it contains only
 # 1 compute node, only intra-node troughput will be measured.
-availability_zone: 'nova'
+# If empty (default), VMTP will automatically pick the first 2 hosts
+# that are compute nodes regardless of the availability zone
+#availability_zone: 'nova'
+availability_zone:
 
 # DNS server IP addresses to use for the VM (list of 1 or more DNS servers)
 # This default DNS server is available on the Internet,

compute.py

@@ -213,23 +213,136 @@ class Compute(object):
         flavor = self.novaclient.flavors.find(name=flavor_type)
         return flavor
 
-    #
+    def normalize_az_host(self, az, host):
-    #   Return a list of hosts which are in a specific availability zone
+        if not az:
-    #   May fail per policy in that case return an empty list
+            az = self.config.availability_zone
-    def list_hypervisor(self, zone_info):
+        return az + ':' + host
-        if self.config.hypervisors:
-            print 'Using hypervisors:' + ', '.join(self.config.hypervisors)
-            return self.config.hypervisors
 
+    def auto_fill_az(self, host_list, host):
+        '''
+        no az provided, if there is a host list we can auto-fill the az
+        else we use the configured az if available
+        else we return an error
+        '''
+        if host_list:
+            for hyp in host_list:
+                if hyp.host_name == host:
+                    return self.normalize_az_host(hyp.zone, host)
+            # no match on host
+            print('Error: passed host name does not exist: ' + host)
+            return None
+        if self.config.availability_zone:
+            return self.normalize_az_host(None, host)
+        print('Error: --hypervisor passed without an az and no az configured')
+        return None
+
+    def sanitize_az_host(self, host_list, az_host):
+        '''
+        host_list: list of hosts as retrieved from openstack (can be empty)
+        az_host: either a host or a az:host string
+        if a host, will check host is in the list, find the corresponding az and
+                    return az:host
+        if az:host is passed will check the host is in the list and az matches
+        if host_list is empty, will return the configured az if there is no
+                    az passed
+        '''
+        if ':' in az_host:
+            # no host_list, return as is (no check)
+            if not host_list:
+                return az_host
+            # if there is a host_list, extract and verify the az and host
+            az_host_list = az_host.split(':')
+            zone = az_host_list[0]
+            host = az_host_list[1]
+            for hyp in host_list:
+                if hyp.host_name == host:
+                    if hyp.zone == zone:
+                        # matches
+                        return az_host
+                    # else continue - another zone with same host name?
+            # no match
+            print('Error: no match for availability zone and host ' + az_host)
+            return None
+        else:
+            return self.auto_fill_az(host_list, az_host)
+
+    #
+    #   Return a list of 0, 1 or 2 az:host
+    #
+    #   The list is computed as follows:
+    #   The list of all hosts is retrieved first from openstack
+    #        if this fails, checks and az auto-fill are disabled
+    #
+    #   If the user provides a list of hypervisors (--hypervisor)
+    #       that list is checked and returned
+    #
+    #   If the user provides a configured az name (config.availability_zone)
+    #       up to the first 2 hosts from the list that match the az are returned
+    #
+    #   If the user did not configure an az name
+    #       up to the first 2 hosts from the list are returned
+    #   Possible return values:
+    #   [ az ]
+    #   [ az:hyp ]
+    #   [ az1:hyp1, az2:hyp2 ]
+    #   []  if an error occurred (error message printed to console)
+    #
+    def get_az_host_list(self):
         avail_list = []
+        host_list = []
+
         try:
             host_list = self.novaclient.hosts.list()
-            for host in host_list:
-                if host.zone == zone_info:
-                    avail_list.append(host.host_name)
         except novaclient.exceptions.Forbidden:
-            print ('Operation Forbidden: could not retrieve list of servers'
+            print ('Warning: Operation Forbidden: could not retrieve list of hosts'
-                   ' in AZ (likely no permission)')
+                   ' (likely no permission)')
+
+        # the user has specified a list of 1 or 2 hypervisors to use
+        if self.config.hypervisors:
+            for hyp in self.config.hypervisors:
+                hyp = self.sanitize_az_host(host_list, hyp)
+                if hyp:
+                    avail_list.append(hyp)
+                else:
+                    return []
+                # if the user did not specify an az, insert the configured az
+                if ':' not in hyp:
+                    if self.config.availability_zone:
+                        hyp = self.normalize_az_host(None, hyp)
+                    else:
+                        return []
+                # pick first 2 matches at most
+                if len(avail_list) == 2:
+                    break
+            print 'Using hypervisors:' + ', '.join(avail_list)
+        else:
+            for host in host_list:
+                # this host must be a compute node
+                if host._info['service'] != 'compute':
+                    continue
+                candidate = None
+                if self.config.availability_zone:
+                    if host.zone == self.config.availability_zone:
+                        candidate = self.normalize_az_host(None, host.host_name)
+                else:
+                    candidate = self.normalize_az_host(host.zone, host.host_name)
+                if candidate:
+                    avail_list.append(candidate)
+                    # pick first 2 matches at most
+                    if len(avail_list) == 2:
+                        break
+
+        # if empty we insert the configured az
+        if not avail_list:
+
+            if not self.config.availability_zone:
+                print('Error: availability_zone must be configured')
+            elif host_list:
+                print('Error: no host matching the selection for availability zone: '
+                      + self.config.availability_zone)
+                avail_list = []
+            else:
+                avail_list = [self.config.availability_zone]
         return avail_list
 
     # Given 2 VMs test if they are running on same Host or not

doc/source/usage.rst

@@ -153,18 +153,26 @@ The default behavior for both TCP/UDP are unlimited. For TCP, we are leveraging
 This is useful when running vmtp on production clouds. The test tool will use up all the bandwidth that may be needed by any other live VMs if we don't set any bandwidth limit. This feature will help to prevent impacting other VMs while running the test tool.
 
 
-Host Selection in Availability Zone
+Host Selection and Availability Zone
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-The *--hypervisor* argument can be used to specify explicitly where to run the test VM in the configured availability zone.
+VMTP requires 1 physical host to perform intra-node tests and 2 hosts to perform inter-node tests.
+There are multiple ways to specify the placement of test VMs to VMTP. By default, VMTP will pick the first 2 compute hosts it can find, regardless of the availability zone.
 
-This can be handy for example when exact VM placement can impact the data path performance (for example rack based placement when the availability zone spans across multiple racks).
+It is possible to limit the host selection to a specific availability zone by specifying its name in the yaml configuration file ('availability_name' parameter).
 
+The *--hypervisor* argument can also be used to specify explicitly on which hosts to run the test VMs.
 The first *--hypervisor* argument specifies on which host to run the test server VM. The second *--hypervisor* argument (in the command line) specifies on which host to run the test client VMs.
+The syntax to use for the argument value is either availability_zone and host name separated by a column (e.g. "--hypervisor nova:host26") or host name (e.g. "--hypervisor host12").
+In the latter case, VMTP will automaticaly pick the availability zone of the host.
 
-The value of the argument must match the hypervisor host name as known by OpenStack (or as displayed using "nova hypervisor-list")
+Picking a particular host can be handy for example when exact VM placement can impact the data path performance (for example rack based placement).
+
+The first --hypervisor argument specifies on which host to run the test server VM. The second --hypervisor argument specifies on which host to run the test client VMs.
+
+The value of the argument must match the hypervisor host name as known by OpenStack (or as displayed using "nova hypervisor-list").
+If an availability zone is provided, VMTP will check that the host name exists in that availability zone.
 
-Example of usage is given below.
 
 Upload Images to Glance
 ^^^^^^^^^^^^^^^^^^^^^^^

requirements.txt

@@ -7,11 +7,14 @@ Babel>=1.3
 
 configure>=0.5
 ecdsa>=0.11
+jsonpatch>=1.9
+jsonschema>=2.4.0
 lxml>=3.4.0
 oslo.utils>=1.2.0
 paramiko>=1.14.0
 pycrypto>=2.6.1
 pymongo>=2.7.2
+python-glanceclient>=0.15.0
 python-neutronclient<3,>=2.3.6
 python-novaclient>=2.18.1
 python-openstackclient>=0.4.1

vmtp.py

@@ -261,28 +261,21 @@ class VmtpTest(object):
             self.assert_true(self.net.vm_int_net)
 
         # Get hosts for the availability zone to use
-        avail_list = self.comp.list_hypervisor(config.availability_zone)
+        # avail_list = self.comp.list_hypervisor(config.availability_zone)
+        avail_list = self.comp.get_az_host_list()
+        if not avail_list:
+            sys.exit(5)
 
         # compute the list of client vm placements to run
-        if avail_list:
+        # the first host is always where the server runs
-            server_az = config.availability_zone + ":" + avail_list[0]
+        server_az = avail_list[0]
         if len(avail_list) > 1:
-                # can do intra + inter
+            # 2 hosts are known
             if config.inter_node_only:
-                    # inter-node only
+                # in this case we do not want the client to run on the same host
-                    self.client_az_list = [config.availability_zone +
+                # as the server
-                                           ":" + avail_list[1]]
+                avail_list.pop(0)
-                else:
+        self.client_az_list = avail_list
-                    self.client_az_list = [server_az, config.availability_zone +
-                                           ":" + avail_list[1]]
-            else:
-                # can only do intra
-                self.client_az_list = [server_az]
-        else:
-            # cannot get the list of hosts
-            # can do intra or inter (cannot know)
-            server_az = config.availability_zone
-            self.client_az_list = [server_az]
 
         self.server = PerfInstance(config.vm_name_server,
                                    config,
@@ -537,8 +530,8 @@ if __name__ == '__main__':
 
     parser.add_argument('--hypervisor', dest='hypervisors',
                         action='append',
-                        help='hypervisor to use in the avail zone (1 per arg, up to 2 args)',
+                        help='hypervisor to use (1 per arg, up to 2 args)',
-                        metavar='name')
+                        metavar='[az:]hostname')
 
     parser.add_argument('--inter-node-only', dest='inter_node_only',
                         default=False,