From 5ac228a575153e059e3d683d5e125415dd32e89f Mon Sep 17 00:00:00 2001 From: mklyus Date: Tue, 17 Nov 2020 18:04:35 +0300 Subject: [PATCH] Bootable volume support (bump) Change-Id: I2989587204766dc1dfc51979608a691578d1f660 --- .zuul.yaml | 11 +++++++ test-requirements.txt | 2 +- vmtp/compute.py | 13 +++++--- vmtp/instance.py | 6 ++-- vmtp/perf_instance.py | 4 +-- vmtp/vmtp.py | 73 +++++++++++++++++++++++++++++++++++++++++-- 6 files changed, 98 insertions(+), 11 deletions(-) create mode 100644 .zuul.yaml diff --git a/.zuul.yaml b/.zuul.yaml new file mode 100644 index 0000000..6b739a0 --- /dev/null +++ b/.zuul.yaml @@ -0,0 +1,11 @@ +- project: + check: + jobs: + - openstack-tox-pep8 + - openstack-tox-py36 + - openstack-tox-docs + gate: + jobs: + - openstack-tox-pep8 + - openstack-tox-py36 + - openstack-tox-docs diff --git a/test-requirements.txt b/test-requirements.txt index a9d909b..34478f3 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -7,11 +7,11 @@ hacking<1.2.0,>=1.1.0 # Apache-2.0 coverage!=4.4 # Apache-2.0 discover python-subunit # Apache-2.0/BSD -sphinx!=1.6.6,!=1.6.7,<2.0.0;python_version=='2.7' # BSD sphinx!=1.6.6,!=1.6.7;python_version>='3.4' # BSD sphinx-rtd-theme>=0.1.9 oslosphinx # Apache-2.0 oslotest # Apache-2.0 +pyflakes==2.1.1 testrepository # Apache-2.0/BSD testscenarios # Apache-2.0/BSD testtools # MIT diff --git a/vmtp/compute.py b/vmtp/compute.py index be9b8d8..9600e30 100644 --- a/vmtp/compute.py +++ b/vmtp/compute.py @@ -149,16 +149,21 @@ class Compute(object): # and check that it gets into the ACTIVE state def create_server(self, vmname, image, flavor, key_name, nic, sec_group, avail_zone=None, user_data=None, - config_drive=None, files=None, retry_count=10): + config_drive=None, files=None, retry_count=10, volume=None): if sec_group: security_groups = [sec_group["name"]] else: security_groups = None - # Also attach the created security group for the test + vol_map = None + if volume: + vol_map = [{"source_type": "volume", "boot_index": "0", + "uuid": volume.id, "destination_type": "volume"}] + image = None instance = self.novaclient.servers.create(name=vmname, image=image, + block_device_mapping_v2=vol_map, flavor=flavor, key_name=key_name, nics=nic, @@ -363,7 +368,7 @@ class Compute(object): if len(avail_list) == 2: break LOG.info('Using hypervisors ' + ', '.join(avail_list)) - else: + if len(avail_list) < 2: for host in host_list: # this host must be a compute node if host.binary != 'nova-compute' or host.state != 'up' or host.status != 'enabled': @@ -374,7 +379,7 @@ class Compute(object): candidate = self.normalize_az_host(None, host.host) else: candidate = self.normalize_az_host(host.zone, host.host) - if candidate: + if candidate and candidate not in avail_list: avail_list.append(candidate) # pick first 2 matches at most if len(avail_list) == 2: diff --git a/vmtp/instance.py b/vmtp/instance.py index 9810906..69c97e1 100644 --- a/vmtp/instance.py +++ b/vmtp/instance.py @@ -146,7 +146,8 @@ class Instance(object): az, internal_network_name, sec_group, - init_file_name=None): + init_file_name=None, + volume=None): # if ssh is created it means this is a native host not a vm if self.ssh: return True @@ -185,7 +186,8 @@ class Instance(object): user_data, self.config_drive, files, - self.config.generic_retry_count) + self.config.generic_retry_count, + volume) if user_data: user_data.close() if not self.instance: diff --git a/vmtp/perf_instance.py b/vmtp/perf_instance.py index 3b8b17e..e8e915d 100644 --- a/vmtp/perf_instance.py +++ b/vmtp/perf_instance.py @@ -36,7 +36,7 @@ class PerfInstance(Instance): ssh_access=None, nics=None, az=None, management_network_name=None, sec_group=None, - init_file_name=None): + init_file_name=None, volume=None): '''Create an instance :return: True on success, False on error ''' @@ -44,7 +44,7 @@ class PerfInstance(Instance): nics, az, management_network_name, sec_group, - init_file_name) + init_file_name, volume) if not rc: return False if self.tp_tool and not self.tp_tool.install(): diff --git a/vmtp/vmtp.py b/vmtp/vmtp.py index 58cfe0b..5b9d457 100755 --- a/vmtp/vmtp.py +++ b/vmtp/vmtp.py @@ -24,6 +24,7 @@ import pprint import re import sys import traceback +import time from .__init__ import __version__ from . import compute @@ -31,8 +32,11 @@ from .config import config_load from .config import config_loads from . import credentials from .fluentd import FluentLogHandler +import cinderclient.exceptions as cinder_exception +from cinderclient.v2 import client as cinderclient from glanceclient.v2 import client as glanceclient from . import iperf_tool +import keystoneauth1 from keystoneclient import client as keystoneclient from .log import CONLOG from .log import FILELOG @@ -159,6 +163,14 @@ class VmtpTest(object): def create_instance(self, inst, az, int_net): fn = self.config.user_data_file user_data_file = fn if fn and os.path.isfile(fn) else None + volume = None + if self.config.volume: + volume = self.create_volume(self.cinder_client, + self.image_instance, + self.config.image_name + "_" + + inst.name + "_" + time.strftime('%d%m_%H%M%S'), + self.config.volume) + self.assert_true(inst.create(self.image_instance, self.flavor_type, self.instance_access, @@ -166,12 +178,49 @@ class VmtpTest(object): az, int_net['name'], self.sec_group, - init_file_name=user_data_file)) + init_file_name=user_data_file, + volume=volume)) def assert_true(self, cond): if not cond: raise VmtpException('Assert failure') + def create_volume(self, cinder_client, image, vol_name, vol_size): + LOG.info('Creating new volume: {}'.format(vol_name)) + retry = 0 + retry_count = 60 + volume = cinder_client.volumes.create(name=str(vol_name), size=vol_size, imageRef=image.id) + while volume.status in ['creating', 'downloading'] and retry < retry_count: + try: + volume = cinder_client.volumes.find(name=volume.name) + except (cinder_exception.NotFound, keystoneauth1.exceptions.http.NotFound): + pass + retry = retry + 1 + LOG.debug("Volume not yet active, retrying %s of %s...", retry, retry_count) + time.sleep(2) + if volume.status != 'available': + raise Exception + return volume + + def delete_volumes(self, cinder_client, volumes): + if isinstance(volumes, str): + volumes = [volumes] + for volume in volumes: + LOG.info('Deleting volume: {}'.format(volume.name)) + cinder_client.volumes.delete(volume) + + def find_volumes(self, cinder_client, volume_name): + res_vol = [] + try: + req_vols = cinder_client.volumes.list() + for vol in req_vols: + if volume_name in vol.name: + res_vol.append(vol) + return res_vol + except (cinder_exception.NotFound, keystoneauth1.exceptions.http.NotFound): + pass + return None + def setup(self): # This is a template host access that will be used for all instances # (the only specific field specific to each instance is the host IP) @@ -208,9 +257,16 @@ class VmtpTest(object): nova_client = novaclient.Client('2', session=sess) neutron = neutronclient.Client('2.0', session=sess) self.glance_client = glanceclient.Client('2', session=sess) + self.cinder_client = cinderclient.Client('2', session=sess) \ + if self.config.volume else None self.comp = compute.Compute(nova_client, neutron, self.config) + if self.config.volume: + volumes = self.find_volumes(self.cinder_client, self.config.image_name) + if volumes: + LOG.info("Removing old VMTP volumes: {}".format(volumes)) + self.delete_volumes(self.cinder_client, volumes) # Add the appropriate public key to openstack self.comp.init_key_pair(self.config.public_key_name, self.instance_access) @@ -295,6 +351,7 @@ class VmtpTest(object): # avail_list = self.comp.list_hypervisor(config.availability_zone) avail_list = self.comp.get_az_host_list() if not avail_list: + self.teardown() sys.exit(5) # compute the list of client vm placements to run @@ -452,6 +509,11 @@ class VmtpTest(object): if not self.config.reuse_existing_vm and self.net: self.net.dispose() # Remove the public key + if self.config.volume: + volumes = self.find_volumes(self.cinder_client, self.config.image_name) + if volumes: + LOG.info("Removing VMTP volumes: {}".format(volumes)) + self.delete_volumes(self.cinder_client, volumes) if self.comp: self.comp.remove_public_key(self.config.public_key_name) # Finally remove the security group @@ -619,7 +681,7 @@ def gen_report_data(proto, result): if proto in ['TCP', 'Upload', 'Download']: for key in list(retval.keys()): if retval[key]: - retval[key] = '{0:n}'.format(retval[key] / tcp_test_count) + retval[key] = '{0:n}'.format(int(retval[key] / tcp_test_count)) else: retval.pop(key) @@ -969,6 +1031,12 @@ def parse_opts_from_cli(): help='Filename for saving VMTP logs', metavar='') + parser.add_argument('--volume', dest='volume', + default=None, + action='store', + help='create bootable volumes for instances', + metavar='') + return parser.parse_known_args()[0] @@ -1012,6 +1080,7 @@ def merge_opts_to_configs(opts): config.keep_first_flow_and_exit = opts.keep_first_flow_and_exit config.inter_node_only = opts.inter_node_only config.same_network_only = opts.same_network_only + config.volume = opts.volume if config.public_key_file and not os.path.isfile(config.public_key_file): LOG.warning('Invalid public_key_file:' + config.public_key_file)