Bootable volume support (bump)

Change-Id: I2989587204766dc1dfc51979608a691578d1f660

.zuul.yaml

@@ -0,0 +1,11 @@
+- project:
+    check:
+      jobs:
+        - openstack-tox-pep8
+        - openstack-tox-py36
+        - openstack-tox-docs
+    gate:
+      jobs:
+        - openstack-tox-pep8
+        - openstack-tox-py36
+        - openstack-tox-docs

test-requirements.txt

@@ -7,11 +7,11 @@ hacking<1.2.0,>=1.1.0 # Apache-2.0
 coverage!=4.4 # Apache-2.0
 discover
 python-subunit # Apache-2.0/BSD
-sphinx!=1.6.6,!=1.6.7,<2.0.0;python_version=='2.7' # BSD
 sphinx!=1.6.6,!=1.6.7;python_version>='3.4' # BSD
 sphinx-rtd-theme>=0.1.9
 oslosphinx # Apache-2.0
 oslotest # Apache-2.0
+pyflakes==2.1.1
 testrepository # Apache-2.0/BSD
 testscenarios # Apache-2.0/BSD
 testtools # MIT

vmtp/compute.py

@@ -149,16 +149,21 @@ class Compute(object):
     # and check that it gets into the ACTIVE state
     def create_server(self, vmname, image, flavor, key_name,
                       nic, sec_group, avail_zone=None, user_data=None,
-                      config_drive=None, files=None, retry_count=10):
+                      config_drive=None, files=None, retry_count=10, volume=None):
 
         if sec_group:
             security_groups = [sec_group["name"]]
         else:
             security_groups = None
-
         # Also attach the created security group for the test
+        vol_map = None
+        if volume:
+            vol_map = [{"source_type": "volume", "boot_index": "0",
+                        "uuid": volume.id, "destination_type": "volume"}]
+            image = None
         instance = self.novaclient.servers.create(name=vmname,
                                                   image=image,
+                                                  block_device_mapping_v2=vol_map,
                                                   flavor=flavor,
                                                   key_name=key_name,
                                                   nics=nic,
@@ -363,7 +368,7 @@ class Compute(object):
                 if len(avail_list) == 2:
                     break
             LOG.info('Using hypervisors ' + ', '.join(avail_list))
-        else:
+        if len(avail_list) < 2:
             for host in host_list:
                 # this host must be a compute node
                 if host.binary != 'nova-compute' or host.state != 'up' or host.status != 'enabled':
@@ -374,7 +379,7 @@ class Compute(object):
                         candidate = self.normalize_az_host(None, host.host)
                 else:
                     candidate = self.normalize_az_host(host.zone, host.host)
-                if candidate:
+                if candidate and candidate not in avail_list:
                     avail_list.append(candidate)
                     # pick first 2 matches at most
                     if len(avail_list) == 2:

vmtp/instance.py

@@ -146,7 +146,8 @@ class Instance(object):
                az,
                internal_network_name,
                sec_group,
-               init_file_name=None):
+               init_file_name=None,
+               volume=None):
         # if ssh is created it means this is a native host not a vm
         if self.ssh:
             return True
@@ -185,7 +186,8 @@ class Instance(object):
                                                 user_data,
                                                 self.config_drive,
                                                 files,
-                                                self.config.generic_retry_count)
+                                                self.config.generic_retry_count,
+                                                volume)
         if user_data:
             user_data.close()
         if not self.instance:

vmtp/perf_instance.py

@@ -36,7 +36,7 @@ class PerfInstance(Instance):
                ssh_access=None, nics=None, az=None,
                management_network_name=None,
                sec_group=None,
-               init_file_name=None):
+               init_file_name=None, volume=None):
         '''Create an instance
         :return: True on success, False on error
         '''
@@ -44,7 +44,7 @@ class PerfInstance(Instance):
                              nics, az,
                              management_network_name,
                              sec_group,
-                             init_file_name)
+                             init_file_name, volume)
         if not rc:
             return False
         if self.tp_tool and not self.tp_tool.install():

vmtp/vmtp.py

@@ -24,6 +24,7 @@ import pprint
 import re
 import sys
 import traceback
+import time
 
 from .__init__ import __version__
 from . import compute
@@ -31,8 +32,11 @@ from .config import config_load
 from .config import config_loads
 from . import credentials
 from .fluentd import FluentLogHandler
+import cinderclient.exceptions as cinder_exception
+from cinderclient.v2 import client as cinderclient
 from glanceclient.v2 import client as glanceclient
 from . import iperf_tool
+import keystoneauth1
 from keystoneclient import client as keystoneclient
 from .log import CONLOG
 from .log import FILELOG
@@ -159,6 +163,14 @@ class VmtpTest(object):
     def create_instance(self, inst, az, int_net):
         fn = self.config.user_data_file
         user_data_file = fn if fn and os.path.isfile(fn) else None
+        volume = None
+        if self.config.volume:
+            volume = self.create_volume(self.cinder_client,
+                                        self.image_instance,
+                                        self.config.image_name + "_" +
+                                        inst.name + "_" + time.strftime('%d%m_%H%M%S'),
+                                        self.config.volume)
+
         self.assert_true(inst.create(self.image_instance,
                                      self.flavor_type,
                                      self.instance_access,
@@ -166,12 +178,49 @@ class VmtpTest(object):
                                      az,
                                      int_net['name'],
                                      self.sec_group,
-                                     init_file_name=user_data_file))
+                                     init_file_name=user_data_file,
+                                     volume=volume))
 
     def assert_true(self, cond):
         if not cond:
             raise VmtpException('Assert failure')
 
+    def create_volume(self, cinder_client, image, vol_name, vol_size):
+        LOG.info('Creating new volume: {}'.format(vol_name))
+        retry = 0
+        retry_count = 60
+        volume = cinder_client.volumes.create(name=str(vol_name), size=vol_size, imageRef=image.id)
+        while volume.status in ['creating', 'downloading'] and retry < retry_count:
+            try:
+                volume = cinder_client.volumes.find(name=volume.name)
+            except (cinder_exception.NotFound, keystoneauth1.exceptions.http.NotFound):
+                pass
+            retry = retry + 1
+            LOG.debug("Volume not yet active, retrying %s of %s...", retry, retry_count)
+            time.sleep(2)
+        if volume.status != 'available':
+            raise Exception
+        return volume
+
+    def delete_volumes(self, cinder_client, volumes):
+        if isinstance(volumes, str):
+            volumes = [volumes]
+        for volume in volumes:
+            LOG.info('Deleting volume: {}'.format(volume.name))
+            cinder_client.volumes.delete(volume)
+
+    def find_volumes(self, cinder_client, volume_name):
+        res_vol = []
+        try:
+            req_vols = cinder_client.volumes.list()
+            for vol in req_vols:
+                if volume_name in vol.name:
+                    res_vol.append(vol)
+            return res_vol
+        except (cinder_exception.NotFound, keystoneauth1.exceptions.http.NotFound):
+            pass
+        return None
+
     def setup(self):
         # This is a template host access that will be used for all instances
         # (the only specific field specific to each instance is the host IP)
@@ -208,9 +257,16 @@ class VmtpTest(object):
             nova_client = novaclient.Client('2', session=sess)
             neutron = neutronclient.Client('2.0', session=sess)
             self.glance_client = glanceclient.Client('2', session=sess)
+            self.cinder_client = cinderclient.Client('2', session=sess) \
+                if self.config.volume else None
 
             self.comp = compute.Compute(nova_client, neutron, self.config)
 
+            if self.config.volume:
+                volumes = self.find_volumes(self.cinder_client, self.config.image_name)
+                if volumes:
+                    LOG.info("Removing old VMTP volumes: {}".format(volumes))
+                    self.delete_volumes(self.cinder_client, volumes)
             # Add the appropriate public key to openstack
             self.comp.init_key_pair(self.config.public_key_name, self.instance_access)
 
@@ -295,6 +351,7 @@ class VmtpTest(object):
         # avail_list = self.comp.list_hypervisor(config.availability_zone)
         avail_list = self.comp.get_az_host_list()
         if not avail_list:
+            self.teardown()
             sys.exit(5)
 
         # compute the list of client vm placements to run
@@ -452,6 +509,11 @@ class VmtpTest(object):
         if not self.config.reuse_existing_vm and self.net:
             self.net.dispose()
         # Remove the public key
+        if self.config.volume:
+            volumes = self.find_volumes(self.cinder_client, self.config.image_name)
+            if volumes:
+                LOG.info("Removing VMTP volumes: {}".format(volumes))
+                self.delete_volumes(self.cinder_client, volumes)
         if self.comp:
             self.comp.remove_public_key(self.config.public_key_name)
         # Finally remove the security group
@@ -619,7 +681,7 @@ def gen_report_data(proto, result):
         if proto in ['TCP', 'Upload', 'Download']:
             for key in list(retval.keys()):
                 if retval[key]:
-                    retval[key] = '{0:n}'.format(retval[key] / tcp_test_count)
+                    retval[key] = '{0:n}'.format(int(retval[key] / tcp_test_count))
                 else:
                     retval.pop(key)
 
@@ -969,6 +1031,12 @@ def parse_opts_from_cli():
                         help='Filename for saving VMTP logs',
                         metavar='<log_file>')
 
+    parser.add_argument('--volume', dest='volume',
+                        default=None,
+                        action='store',
+                        help='create bootable volumes for instances',
+                        metavar='<volume>')
+
     return parser.parse_known_args()[0]
 
 
@@ -1012,6 +1080,7 @@ def merge_opts_to_configs(opts):
     config.keep_first_flow_and_exit = opts.keep_first_flow_and_exit
     config.inter_node_only = opts.inter_node_only
     config.same_network_only = opts.same_network_only
+    config.volume = opts.volume
 
     if config.public_key_file and not os.path.isfile(config.public_key_file):
         LOG.warning('Invalid public_key_file:' + config.public_key_file)