diff --git a/requirements.txt b/requirements.txt index 2c6b4bd..0ed7dc9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ prettytable>=0.7.2 pymongo>=2.7.2 pytz>=2016.4 python-glanceclient>=0.15.0 -python-neutronclient<3,>=2.3.6 +python-neutronclient>=3.0.0 python-novaclient>=2.18.1 python-openstackclient>=0.4.1 python-keystoneclient>=1.7.2 diff --git a/vmtp/compute.py b/vmtp/compute.py index 8009aa0..bcb580a 100644 --- a/vmtp/compute.py +++ b/vmtp/compute.py @@ -19,22 +19,30 @@ import os import time import glanceclient.exc as glance_exception +import keystoneauth1 from log import LOG import novaclient import novaclient.exceptions as exceptions -class Compute(object): +try: + from glanceclient.openstack.common.apiclient.exceptions import NotFound as GlanceImageNotFound +except ImportError: + from glanceclient.v1.apiclient.exceptions import NotFound as GlanceImageNotFound - def __init__(self, nova_client, config): + +class Compute(object): + def __init__(self, nova_client, neutron, config): self.novaclient = nova_client + self.neutron = neutron self.config = config - def find_image(self, image_name): + def find_image(self, glance_client, image_name): try: - image = self.novaclient.images.find(name=image_name) - return image - except novaclient.exceptions.NotFound: - return None + return next(glance_client.images.list(filters={'name': image_name}), None) + except (novaclient.exceptions.NotFound, keystoneauth1.exceptions.http.NotFound, + GlanceImageNotFound): + pass + return None def upload_image_via_url(self, glance_client, final_image_name, image_url, retry_count=60): @@ -42,25 +50,22 @@ class Compute(object): Directly uploads image to Nova via URL if image is not present ''' retry = 0 + file_prefix = "file://" + if not image_url.startswith(file_prefix): + LOG.error("File format %s is not supported. It must start with %s", image_url, + file_prefix) + return False try: - # check image is file/url based. - file_prefix = "file://" - if image_url.startswith(file_prefix): - image_location = image_url.split(file_prefix)[1] - with open(image_location) as f_image: - img = glance_client.images.create( - name=str(final_image_name), disk_format="qcow2", - container_format="bare", is_public=True, data=f_image) - else: - # Upload the image - img = glance_client.images.create( - name=str(final_image_name), disk_format="qcow2", - container_format="bare", is_public=True, - copy_from=image_url) + image_location = image_url.split(file_prefix)[1] + f_image = open(image_location, 'rb') + img = glance_client.images.create(name=str(final_image_name)) + glance_client.images.update(img.id, disk_format="qcow2", + container_format="bare") + glance_client.images.upload(img.id, f_image) # Check for the image in glance while img.status in ['queued', 'saving'] and retry < retry_count: - img = glance_client.images.find(name=img.name) + img = self.find_image(glance_client, image_name=img.name) retry = retry + 1 LOG.debug("Image not yet active, retrying %s of %s...", retry, retry_count) time.sleep(2) @@ -70,21 +75,17 @@ class Compute(object): LOG.error("Cannot upload image without admin access. Please make " "sure the image is uploaded and is either public or owned by you.") return False - except IOError: + except (Exception, IOError): # catch the exception for file based errors. LOG.error("Failed while uploading the image. Please make sure the " "image at the specified location %s is correct.", image_url) return False - except Exception: - LOG.error("Failed while uploading the image, please make sure the " - "cloud under test has the access to URL: %s.", image_url) - return False return True def delete_image(self, glance_client, img_name): try: LOG.log("Deleting image %s...", img_name) - img = glance_client.images.find(name=img_name) + img = self.find_image(glance_client, img_name) glance_client.images.delete(img.id) except Exception: LOG.error("Failed to delete the image %s.", img_name) @@ -150,7 +151,7 @@ class Compute(object): config_drive=None, files=None, retry_count=10): if sec_group: - security_groups = [sec_group.name] + security_groups = [sec_group["name"]] else: security_groups = None @@ -300,7 +301,7 @@ class Compute(object): if hyp.zone == zone: # matches return az_host - # else continue - another zone with same host name? + # else continue - another zone with same host name? # no match LOG.error('No match for availability zone and host ' + az_host) return None @@ -401,72 +402,85 @@ class Compute(object): # Create a new security group with appropriate rules def security_group_create(self): # check first the security group exists - # May throw exceptions.NoUniqueMatch or NotFound - try: - group = self.novaclient.security_groups.find(name=self.config.security_group_name) - return group - except exceptions.NotFound: - group = self.novaclient.security_groups.create(name=self.config.security_group_name, - description="PNS Security group") - # Once security group try to find it iteratively - # (this check may no longer be necessary) - for _ in range(self.config.generic_retry_count): - group = self.novaclient.security_groups.get(group) - if group: - self.security_group_add_rules(group) - return group - else: - time.sleep(1) - return None - # except exceptions.NoUniqueMatch as exc: - # raise exc + for security_group in self.neutron.list_security_groups()["security_groups"]: + if security_group["name"] == self.config.security_group_name: + return security_group + group = self.neutron.create_security_group({ + "security_group": { + "name": self.config.security_group_name, + "description": "PNS Security groups"}}) + self.security_group_add_rules(group["security_group"]) + return group["security_group"] # Delete a security group def security_group_delete(self, group): if group: LOG.info("Deleting security group") - self.novaclient.security_groups.delete(group) + self.neutron.delete_security_group(group["id"]) # Add rules to the security group def security_group_add_rules(self, group): + # Allow ping traffic - self.novaclient.security_group_rules.create(group.id, - ip_protocol="icmp", - from_port=-1, - to_port=-1) + self.neutron.create_security_group_rule( + self.generate_security_group_rule_dict(group_id=group["id"], + protocol="icmp")) + if self.config.ipv6_mode: - self.novaclient.security_group_rules.create(group.id, - ip_protocol="icmp", - from_port=-1, - to_port=-1, - cidr="::/0") + self.neutron.create_security_group_rule( + self.generate_security_group_rule_dict(group_id=group["id"], + protocol="icmp", + ether_type="IPv6")) + # Allow SSH traffic - self.novaclient.security_group_rules.create(group.id, - ip_protocol="tcp", - from_port=22, - to_port=22) + self.neutron.create_security_group_rule( + self.generate_security_group_rule_dict(group_id=group["id"], + protocol="tcp", + port_range_min=22, + port_range_max=22)) + # Allow TCP/UDP traffic for perf tools like iperf/nuttcp # 5001: Data traffic (standard iperf data port) # 5002: Control traffic (non standard) # note that 5000/tcp is already picked by openstack keystone if not self.config.ipv6_mode: - self.novaclient.security_group_rules.create(group.id, - ip_protocol="tcp", - from_port=5001, - to_port=5002) - self.novaclient.security_group_rules.create(group.id, - ip_protocol="udp", - from_port=5001, - to_port=5001) + self.neutron.create_security_group_rule( + self.generate_security_group_rule_dict(group_id=group["id"], + protocol="tcp", + port_range_min=5001, + port_range_max=5002)) + self.neutron.create_security_group_rule( + self.generate_security_group_rule_dict(group_id=group["id"], + protocol="udp", + port_range_min=5001, + port_range_max=5002)) + else: # IPV6 rules addition - self.novaclient.security_group_rules.create(group.id, - ip_protocol="tcp", - from_port=5001, - to_port=5002, - cidr="::/0") - self.novaclient.security_group_rules.create(group.id, - ip_protocol="udp", - from_port=5001, - to_port=5001, - cidr="::/0") + self.neutron.create_security_group_rule( + self.generate_security_group_rule_dict(group_id=group["id"], + protocol="tcp", + ethertype="IPv6", + port_range_min=5001, + port_range_max=5002)) + self.neutron.create_security_group_rule( + self.generate_security_group_rule_dict(group_id=group["id"], + protocol="udp", + ethertype="IPv6", + port_range_min=5001, + port_range_max=5002)) + + # Generates dict for security group rule + def generate_security_group_rule_dict(self, group_id, protocol, ethertype='IPv4', + port_range_min=None, + port_range_max=None): + return { + 'security_group_rule': { + 'direction': 'ingress', + 'security_group_id': group_id, + 'ethertype': ethertype, + 'port_range_min': port_range_min, + 'port_range_max': port_range_max, + 'protocol': protocol, + 'remote_group_id': None, + 'remote_ip_prefix': '0.0.0.0/0'}} diff --git a/vmtp/vmtp.py b/vmtp/vmtp.py index 4f881e4..d567ac0 100755 --- a/vmtp/vmtp.py +++ b/vmtp/vmtp.py @@ -30,7 +30,7 @@ import compute from config import config_load from config import config_loads import credentials -from glanceclient import client as glanceclient +from glanceclient.v2 import client as glanceclient import iperf_tool from keystoneclient.auth.identity import v2 as keystone_v2 from keystoneclient.auth.identity import v3 as keystone_v3 @@ -52,6 +52,8 @@ import sshutils flow_num = 0 return_code = 0 + + class FlowPrinter(object): @staticmethod def print_desc(desc): @@ -60,8 +62,8 @@ class FlowPrinter(object): CONLOG.info("=" * 60) LOG.info('Flow %d: %s', flow_num, desc) -class ResultsCollector(object): +class ResultsCollector(object): def __init__(self): self.results = {'flows': []} self.results['date'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') @@ -109,7 +111,7 @@ class ResultsCollector(object): def save_to_db(self, cfg): '''Save results to MongoDB database.''' LOG.info("Saving results to MongoDB database...") - post_id = pns_mongo.\ + post_id = pns_mongo. \ pns_add_test_result_to_mongod(cfg.vmtp_mongod_ip, cfg.vmtp_mongod_port, cfg.vmtp_db, @@ -118,9 +120,11 @@ class ResultsCollector(object): if post_id is None: LOG.error("Failed to add result to DB") + class VmtpException(Exception): pass + class VmtpTest(object): def __init__(self, config, cred, rescol): ''' @@ -209,23 +213,25 @@ class VmtpTest(object): # Create the nova and neutron instances nova_client = novaclient.Client('2', session=sess) neutron = neutronclient.Client('2.0', session=sess) + self.glance_client = glanceclient.Client('2', session=sess) - self.comp = compute.Compute(nova_client, self.config) + self.comp = compute.Compute(nova_client, neutron, self.config) # Add the appropriate public key to openstack self.comp.init_key_pair(self.config.public_key_name, self.instance_access) - self.image_instance = self.comp.find_image(self.config.image_name) + self.image_instance = self.comp.find_image(self.glance_client, self.config.image_name) if self.image_instance is None: + print(len(self.config.vm_image_url)) if self.config.vm_image_url != "": LOG.info('%s: image for VM not found, trying to upload it ...', self.config.image_name) - self.glance_client = glanceclient.Client('1', session=sess) self.comp.upload_image_via_url( self.glance_client, self.config.image_name, self.config.vm_image_url) - self.image_instance = self.comp.find_image(self.config.image_name) + self.image_instance = self.comp.find_image(self.glance_client, + self.config.image_name) self.image_uploaded = True else: # Exit the pogram @@ -478,6 +484,7 @@ class VmtpTest(object): else: self.teardown() + def test_native_tp(nhosts, ifname, config): FlowPrinter.print_desc('Native Host to Host throughput') result_list = [] @@ -532,6 +539,7 @@ def test_native_tp(nhosts, ifname, config): return result_list + def get_controller_info(ssh_access, net, res_col, retry_count): if not ssh_access: return @@ -558,6 +566,7 @@ def get_controller_info(ssh_access, net, res_col, retry_count): res_col.add_properties(res) + def gen_report_data(proto, result): try: if proto in ['TCP', 'UDP', 'Multicast', 'ICMP']: @@ -607,8 +616,8 @@ def gen_report_data(proto, result): traceback.print_exc() return retval -def print_report(results): +def print_report(results): # In order to parse the results with less logic, we are encoding the results as below: # Same Network = 0, Different Network = 1 # Fixed IP = 0, Floating IP = 1 @@ -708,6 +717,7 @@ def print_report(results): global return_code return_code = 1 + def normalize_paths(cfg): ''' Normalize the various paths to config files, tools, ssh priv and pub key @@ -721,6 +731,7 @@ def normalize_paths(cfg): if cfg.private_key_file: cfg.private_key_file = os.path.expanduser(os.path.expanduser(cfg.private_key_file)) + def get_ssh_access(opt_name, opt_value, config): '''Allocate a HostSshAccess instance to the option value Check that a password is provided or the key pair in the config file @@ -738,6 +749,7 @@ def get_ssh_access(opt_name, opt_value, config): sys.exit(2) return host_access + def parse_opts_from_cli(): parser = argparse.ArgumentParser() parser.add_argument('-c', '--config', dest='config', @@ -947,6 +959,7 @@ def parse_opts_from_cli(): return parser.parse_known_args()[0] + def decode_size_list(argname, size_list): try: pkt_sizes = size_list.split(',') @@ -958,8 +971,8 @@ def decode_size_list(argname, size_list): sys.exit(1) return pkt_sizes -def merge_opts_to_configs(opts): +def merge_opts_to_configs(opts): default_cfg_file = resource_string(__name__, "cfg.default.yaml") # read the default configuration file and possibly an override config file # the precedence order is as follows: @@ -1056,7 +1069,6 @@ def merge_opts_to_configs(opts): sys.exit(1) config.vm_bandwidth = int(val * (10 ** (ex_unit * 3))) - # the pkt size for TCP, UDP and ICMP if opts.tcp_pkt_sizes: config.tcp_pkt_sizes = decode_size_list('--tcpbuf', opts.tcp_pkt_sizes) @@ -1130,6 +1142,7 @@ def merge_opts_to_configs(opts): return config + def run_vmtp(opts): '''Run VMTP :param opts: Parameters that to be passed to VMTP in type argparse.Namespace(). See: @@ -1214,11 +1227,13 @@ def run_vmtp(opts): return rescol.results + def main(): opts = parse_opts_from_cli() log.setup('vmtp', debug=opts.debug, logfile=opts.logfile) run_vmtp(opts) sys.exit(return_code) + if __name__ == '__main__': main()