x/vmtp
Code Issues Proposed changes

Fixed nova api to neutron for security groups and nova api to glance for image uploading

Browse Source 
Change-Id: Ia868d517b8295eba4afa2f12d88ccb4e0e1c249d
This commit is contained in:
Kerim Gokarslan 2017-08-17 14:54:08 -07:00
parent e88b7f77c8
commit e5977e6f1e
3 changed files with 121 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
requirements.txt
View File

@ -12,7 +12,7 @@ prettytable>=0.7.2
pymongo>=2.7.2 pymongo>=2.7.2
pytz>=2016.4 pytz>=2016.4
python-glanceclient>=0.15.0 python-glanceclient>=0.15.0
python-neutronclient<3,>=2.3.6 python-neutronclient>=3.0.0
python-novaclient>=2.18.1 python-novaclient>=2.18.1
python-openstackclient>=0.4.1 python-openstackclient>=0.4.1
python-keystoneclient>=1.7.2 python-keystoneclient>=1.7.2

176
vmtp/compute.py
View File

@ -19,22 +19,30 @@ import os
import time import time
import glanceclient.exc as glance_exception import glanceclient.exc as glance_exception
import keystoneauth1
from log import LOG from log import LOG
import novaclient import novaclient
import novaclient.exceptions as exceptions import novaclient.exceptions as exceptions
class Compute(object): try:
from glanceclient.openstack.common.apiclient.exceptions import NotFound as GlanceImageNotFound
except ImportError:
from glanceclient.v1.apiclient.exceptions import NotFound as GlanceImageNotFound
def __init__(self, nova_client, config):
class Compute(object):
def __init__(self, nova_client, neutron, config):
self.novaclient = nova_client self.novaclient = nova_client
self.neutron = neutron
self.config = config self.config = config
def find_image(self, image_name): def find_image(self, glance_client, image_name):
try: try:
image = self.novaclient.images.find(name=image_name) return next(glance_client.images.list(filters={'name': image_name}), None)
return image except (novaclient.exceptions.NotFound, keystoneauth1.exceptions.http.NotFound,
except novaclient.exceptions.NotFound: GlanceImageNotFound):
return None pass
return None
def upload_image_via_url(self, glance_client, final_image_name, def upload_image_via_url(self, glance_client, final_image_name,
image_url, retry_count=60): image_url, retry_count=60):
@ -42,25 +50,22 @@ class Compute(object):
Directly uploads image to Nova via URL if image is not present Directly uploads image to Nova via URL if image is not present
''' '''
retry = 0 retry = 0
file_prefix = "file://"
if not image_url.startswith(file_prefix):
LOG.error("File format %s is not supported. It must start with %s", image_url,
file_prefix)
return False
try: try:
# check image is file/url based. image_location = image_url.split(file_prefix)[1]
file_prefix = "file://" f_image = open(image_location, 'rb')
if image_url.startswith(file_prefix): img = glance_client.images.create(name=str(final_image_name))
image_location = image_url.split(file_prefix)[1] glance_client.images.update(img.id, disk_format="qcow2",
with open(image_location) as f_image: container_format="bare")
img = glance_client.images.create( glance_client.images.upload(img.id, f_image)
name=str(final_image_name), disk_format="qcow2",
container_format="bare", is_public=True, data=f_image)
else:
# Upload the image
img = glance_client.images.create(
name=str(final_image_name), disk_format="qcow2",
container_format="bare", is_public=True,
copy_from=image_url)
# Check for the image in glance # Check for the image in glance
while img.status in ['queued', 'saving'] and retry < retry_count: while img.status in ['queued', 'saving'] and retry < retry_count:
img = glance_client.images.find(name=img.name) img = self.find_image(glance_client, image_name=img.name)
retry = retry + 1 retry = retry + 1
LOG.debug("Image not yet active, retrying %s of %s...", retry, retry_count) LOG.debug("Image not yet active, retrying %s of %s...", retry, retry_count)
time.sleep(2) time.sleep(2)
@ -70,21 +75,17 @@ class Compute(object):
LOG.error("Cannot upload image without admin access. Please make " LOG.error("Cannot upload image without admin access. Please make "
"sure the image is uploaded and is either public or owned by you.") "sure the image is uploaded and is either public or owned by you.")
return False return False
except IOError: except (Exception, IOError):
# catch the exception for file based errors. # catch the exception for file based errors.
LOG.error("Failed while uploading the image. Please make sure the " LOG.error("Failed while uploading the image. Please make sure the "
"image at the specified location %s is correct.", image_url) "image at the specified location %s is correct.", image_url)
return False return False
except Exception:
LOG.error("Failed while uploading the image, please make sure the "
"cloud under test has the access to URL: %s.", image_url)
return False
return True return True
def delete_image(self, glance_client, img_name): def delete_image(self, glance_client, img_name):
try: try:
LOG.log("Deleting image %s...", img_name) LOG.log("Deleting image %s...", img_name)
img = glance_client.images.find(name=img_name) img = self.find_image(glance_client, img_name)
glance_client.images.delete(img.id) glance_client.images.delete(img.id)
except Exception: except Exception:
LOG.error("Failed to delete the image %s.", img_name) LOG.error("Failed to delete the image %s.", img_name)
@ -150,7 +151,7 @@ class Compute(object):
config_drive=None, files=None, retry_count=10): config_drive=None, files=None, retry_count=10):
if sec_group: if sec_group:
security_groups = [sec_group.name] security_groups = [sec_group["name"]]
else: else:
security_groups = None security_groups = None
@ -300,7 +301,7 @@ class Compute(object):
if hyp.zone == zone: if hyp.zone == zone:
# matches # matches
return az_host return az_host
# else continue - another zone with same host name? # else continue - another zone with same host name?
# no match # no match
LOG.error('No match for availability zone and host ' + az_host) LOG.error('No match for availability zone and host ' + az_host)
return None return None
@ -401,72 +402,85 @@ class Compute(object):
# Create a new security group with appropriate rules # Create a new security group with appropriate rules
def security_group_create(self): def security_group_create(self):
# check first the security group exists # check first the security group exists
# May throw exceptions.NoUniqueMatch or NotFound for security_group in self.neutron.list_security_groups()["security_groups"]:
try: if security_group["name"] == self.config.security_group_name:
group = self.novaclient.security_groups.find(name=self.config.security_group_name) return security_group
return group group = self.neutron.create_security_group({
except exceptions.NotFound: "security_group": {
group = self.novaclient.security_groups.create(name=self.config.security_group_name, "name": self.config.security_group_name,
description="PNS Security group") "description": "PNS Security groups"}})
# Once security group try to find it iteratively self.security_group_add_rules(group["security_group"])
# (this check may no longer be necessary) return group["security_group"]
for _ in range(self.config.generic_retry_count):
group = self.novaclient.security_groups.get(group)
if group:
self.security_group_add_rules(group)
return group
else:
time.sleep(1)
return None
# except exceptions.NoUniqueMatch as exc:
# raise exc
# Delete a security group # Delete a security group
def security_group_delete(self, group): def security_group_delete(self, group):
if group: if group:
LOG.info("Deleting security group") LOG.info("Deleting security group")
self.novaclient.security_groups.delete(group) self.neutron.delete_security_group(group["id"])
# Add rules to the security group # Add rules to the security group
def security_group_add_rules(self, group): def security_group_add_rules(self, group):
# Allow ping traffic # Allow ping traffic
self.novaclient.security_group_rules.create(group.id, self.neutron.create_security_group_rule(
ip_protocol="icmp", self.generate_security_group_rule_dict(group_id=group["id"],
from_port=-1, protocol="icmp"))
to_port=-1)
if self.config.ipv6_mode: if self.config.ipv6_mode:
self.novaclient.security_group_rules.create(group.id, self.neutron.create_security_group_rule(
ip_protocol="icmp", self.generate_security_group_rule_dict(group_id=group["id"],
from_port=-1, protocol="icmp",
to_port=-1, ether_type="IPv6"))
cidr="::/0")
# Allow SSH traffic # Allow SSH traffic
self.novaclient.security_group_rules.create(group.id, self.neutron.create_security_group_rule(
ip_protocol="tcp", self.generate_security_group_rule_dict(group_id=group["id"],
from_port=22, protocol="tcp",
to_port=22) port_range_min=22,
port_range_max=22))
# Allow TCP/UDP traffic for perf tools like iperf/nuttcp # Allow TCP/UDP traffic for perf tools like iperf/nuttcp
# 5001: Data traffic (standard iperf data port) # 5001: Data traffic (standard iperf data port)
# 5002: Control traffic (non standard) # 5002: Control traffic (non standard)
# note that 5000/tcp is already picked by openstack keystone # note that 5000/tcp is already picked by openstack keystone
if not self.config.ipv6_mode: if not self.config.ipv6_mode:
self.novaclient.security_group_rules.create(group.id, self.neutron.create_security_group_rule(
ip_protocol="tcp", self.generate_security_group_rule_dict(group_id=group["id"],
from_port=5001, protocol="tcp",
to_port=5002) port_range_min=5001,
self.novaclient.security_group_rules.create(group.id, port_range_max=5002))
ip_protocol="udp", self.neutron.create_security_group_rule(
from_port=5001, self.generate_security_group_rule_dict(group_id=group["id"],
to_port=5001) protocol="udp",
port_range_min=5001,
port_range_max=5002))
else: else:
# IPV6 rules addition # IPV6 rules addition
self.novaclient.security_group_rules.create(group.id, self.neutron.create_security_group_rule(
ip_protocol="tcp", self.generate_security_group_rule_dict(group_id=group["id"],
from_port=5001, protocol="tcp",
to_port=5002, ethertype="IPv6",
cidr="::/0") port_range_min=5001,
self.novaclient.security_group_rules.create(group.id, port_range_max=5002))
ip_protocol="udp", self.neutron.create_security_group_rule(
from_port=5001, self.generate_security_group_rule_dict(group_id=group["id"],
to_port=5001, protocol="udp",
cidr="::/0") ethertype="IPv6",
port_range_min=5001,
port_range_max=5002))
# Generates dict for security group rule
def generate_security_group_rule_dict(self, group_id, protocol, ethertype='IPv4',
port_range_min=None,
port_range_max=None):
return {
'security_group_rule': {
'direction': 'ingress',
'security_group_id': group_id,
'ethertype': ethertype,
'port_range_min': port_range_min,
'port_range_max': port_range_max,
'protocol': protocol,
'remote_group_id': None,
'remote_ip_prefix': '0.0.0.0/0'}}

35
vmtp/vmtp.py
View File

@ -30,7 +30,7 @@ import compute
from config import config_load from config import config_load
from config import config_loads from config import config_loads
import credentials import credentials
from glanceclient import client as glanceclient from glanceclient.v2 import client as glanceclient
import iperf_tool import iperf_tool
from keystoneclient.auth.identity import v2 as keystone_v2 from keystoneclient.auth.identity import v2 as keystone_v2
from keystoneclient.auth.identity import v3 as keystone_v3 from keystoneclient.auth.identity import v3 as keystone_v3
@ -52,6 +52,8 @@ import sshutils
flow_num = 0 flow_num = 0
return_code = 0 return_code = 0
class FlowPrinter(object): class FlowPrinter(object):
@staticmethod @staticmethod
def print_desc(desc): def print_desc(desc):
@ -60,8 +62,8 @@ class FlowPrinter(object):
CONLOG.info("=" * 60) CONLOG.info("=" * 60)
LOG.info('Flow %d: %s', flow_num, desc) LOG.info('Flow %d: %s', flow_num, desc)
class ResultsCollector(object):
class ResultsCollector(object):
def __init__(self): def __init__(self):
self.results = {'flows': []} self.results = {'flows': []}
self.results['date'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') self.results['date'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
@ -109,7 +111,7 @@ class ResultsCollector(object):
def save_to_db(self, cfg): def save_to_db(self, cfg):
'''Save results to MongoDB database.''' '''Save results to MongoDB database.'''
LOG.info("Saving results to MongoDB database...") LOG.info("Saving results to MongoDB database...")
post_id = pns_mongo.\ post_id = pns_mongo. \
pns_add_test_result_to_mongod(cfg.vmtp_mongod_ip, pns_add_test_result_to_mongod(cfg.vmtp_mongod_ip,
cfg.vmtp_mongod_port, cfg.vmtp_mongod_port,
cfg.vmtp_db, cfg.vmtp_db,
@ -118,9 +120,11 @@ class ResultsCollector(object):
if post_id is None: if post_id is None:
LOG.error("Failed to add result to DB") LOG.error("Failed to add result to DB")
class VmtpException(Exception): class VmtpException(Exception):
pass pass
class VmtpTest(object): class VmtpTest(object):
def __init__(self, config, cred, rescol): def __init__(self, config, cred, rescol):
''' '''
@ -209,23 +213,25 @@ class VmtpTest(object):
# Create the nova and neutron instances # Create the nova and neutron instances
nova_client = novaclient.Client('2', session=sess) nova_client = novaclient.Client('2', session=sess)
neutron = neutronclient.Client('2.0', session=sess) neutron = neutronclient.Client('2.0', session=sess)
self.glance_client = glanceclient.Client('2', session=sess)
self.comp = compute.Compute(nova_client, self.config) self.comp = compute.Compute(nova_client, neutron, self.config)
# Add the appropriate public key to openstack # Add the appropriate public key to openstack
self.comp.init_key_pair(self.config.public_key_name, self.instance_access) self.comp.init_key_pair(self.config.public_key_name, self.instance_access)
self.image_instance = self.comp.find_image(self.config.image_name) self.image_instance = self.comp.find_image(self.glance_client, self.config.image_name)
if self.image_instance is None: if self.image_instance is None:
print(len(self.config.vm_image_url))
if self.config.vm_image_url != "": if self.config.vm_image_url != "":
LOG.info('%s: image for VM not found, trying to upload it ...', LOG.info('%s: image for VM not found, trying to upload it ...',
self.config.image_name) self.config.image_name)
self.glance_client = glanceclient.Client('1', session=sess)
self.comp.upload_image_via_url( self.comp.upload_image_via_url(
self.glance_client, self.glance_client,
self.config.image_name, self.config.image_name,
self.config.vm_image_url) self.config.vm_image_url)
self.image_instance = self.comp.find_image(self.config.image_name) self.image_instance = self.comp.find_image(self.glance_client,
self.config.image_name)
self.image_uploaded = True self.image_uploaded = True
else: else:
# Exit the pogram # Exit the pogram
@ -478,6 +484,7 @@ class VmtpTest(object):
else: else:
self.teardown() self.teardown()
def test_native_tp(nhosts, ifname, config): def test_native_tp(nhosts, ifname, config):
FlowPrinter.print_desc('Native Host to Host throughput') FlowPrinter.print_desc('Native Host to Host throughput')
result_list = [] result_list = []
@ -532,6 +539,7 @@ def test_native_tp(nhosts, ifname, config):
return result_list return result_list
def get_controller_info(ssh_access, net, res_col, retry_count): def get_controller_info(ssh_access, net, res_col, retry_count):
if not ssh_access: if not ssh_access:
return return
@ -558,6 +566,7 @@ def get_controller_info(ssh_access, net, res_col, retry_count):
res_col.add_properties(res) res_col.add_properties(res)
def gen_report_data(proto, result): def gen_report_data(proto, result):
try: try:
if proto in ['TCP', 'UDP', 'Multicast', 'ICMP']: if proto in ['TCP', 'UDP', 'Multicast', 'ICMP']:
@ -607,8 +616,8 @@ def gen_report_data(proto, result):
traceback.print_exc() traceback.print_exc()
return retval return retval
def print_report(results):
def print_report(results):
# In order to parse the results with less logic, we are encoding the results as below: # In order to parse the results with less logic, we are encoding the results as below:
# Same Network = 0, Different Network = 1 # Same Network = 0, Different Network = 1
# Fixed IP = 0, Floating IP = 1 # Fixed IP = 0, Floating IP = 1
@ -708,6 +717,7 @@ def print_report(results):
global return_code global return_code
return_code = 1 return_code = 1
def normalize_paths(cfg): def normalize_paths(cfg):
''' '''
Normalize the various paths to config files, tools, ssh priv and pub key Normalize the various paths to config files, tools, ssh priv and pub key
@ -721,6 +731,7 @@ def normalize_paths(cfg):
if cfg.private_key_file: if cfg.private_key_file:
cfg.private_key_file = os.path.expanduser(os.path.expanduser(cfg.private_key_file)) cfg.private_key_file = os.path.expanduser(os.path.expanduser(cfg.private_key_file))
def get_ssh_access(opt_name, opt_value, config): def get_ssh_access(opt_name, opt_value, config):
'''Allocate a HostSshAccess instance to the option value '''Allocate a HostSshAccess instance to the option value
Check that a password is provided or the key pair in the config file Check that a password is provided or the key pair in the config file
@ -738,6 +749,7 @@ def get_ssh_access(opt_name, opt_value, config):
sys.exit(2) sys.exit(2)
return host_access return host_access
def parse_opts_from_cli(): def parse_opts_from_cli():
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('-c', '--config', dest='config', parser.add_argument('-c', '--config', dest='config',
@ -947,6 +959,7 @@ def parse_opts_from_cli():
return parser.parse_known_args()[0] return parser.parse_known_args()[0]
def decode_size_list(argname, size_list): def decode_size_list(argname, size_list):
try: try:
pkt_sizes = size_list.split(',') pkt_sizes = size_list.split(',')
@ -958,8 +971,8 @@ def decode_size_list(argname, size_list):
sys.exit(1) sys.exit(1)
return pkt_sizes return pkt_sizes
def merge_opts_to_configs(opts):
def merge_opts_to_configs(opts):
default_cfg_file = resource_string(__name__, "cfg.default.yaml") default_cfg_file = resource_string(__name__, "cfg.default.yaml")
# read the default configuration file and possibly an override config file # read the default configuration file and possibly an override config file
# the precedence order is as follows: # the precedence order is as follows:
@ -1056,7 +1069,6 @@ def merge_opts_to_configs(opts):
sys.exit(1) sys.exit(1)
config.vm_bandwidth = int(val * (10 ** (ex_unit * 3))) config.vm_bandwidth = int(val * (10 ** (ex_unit * 3)))
# the pkt size for TCP, UDP and ICMP # the pkt size for TCP, UDP and ICMP
if opts.tcp_pkt_sizes: if opts.tcp_pkt_sizes:
config.tcp_pkt_sizes = decode_size_list('--tcpbuf', opts.tcp_pkt_sizes) config.tcp_pkt_sizes = decode_size_list('--tcpbuf', opts.tcp_pkt_sizes)
@ -1130,6 +1142,7 @@ def merge_opts_to_configs(opts):
return config return config
def run_vmtp(opts): def run_vmtp(opts):
'''Run VMTP '''Run VMTP
:param opts: Parameters that to be passed to VMTP in type argparse.Namespace(). See: :param opts: Parameters that to be passed to VMTP in type argparse.Namespace(). See:
@ -1214,11 +1227,13 @@ def run_vmtp(opts):
return rescol.results return rescol.results
def main(): def main():
opts = parse_opts_from_cli() opts = parse_opts_from_cli()
log.setup('vmtp', debug=opts.debug, logfile=opts.logfile) log.setup('vmtp', debug=opts.debug, logfile=opts.logfile)
run_vmtp(opts) run_vmtp(opts)
sys.exit(return_code) sys.exit(return_code)
if __name__ == '__main__': if __name__ == '__main__':
main() main()