x/vmtp
Code Issues Proposed changes

Fixed nova api to neutron for security groups and nova api to glance for image uploading

Browse Source 
Change-Id: Ia868d517b8295eba4afa2f12d88ccb4e0e1c249d
This commit is contained in:
Kerim Gokarslan 2017-08-17 14:54:08 -07:00
parent e88b7f77c8
commit e5977e6f1e
3 changed files with 121 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
requirements.txt
View File

@ -12,7 +12,7 @@ prettytable>=0.7.2
pymongo>=2.7.2
pytz>=2016.4
python-glanceclient>=0.15.0
python-neutronclient<3,>=2.3.6
python-neutronclient>=3.0.0
python-novaclient>=2.18.1
python-openstackclient>=0.4.1
python-keystoneclient>=1.7.2

176
vmtp/compute.py
View File

@ -19,22 +19,30 @@ import os
import time
import glanceclient.exc as glance_exception
import keystoneauth1
from log import LOG
import novaclient
import novaclient.exceptions as exceptions
class Compute(object):
try:
from glanceclient.openstack.common.apiclient.exceptions import NotFound as GlanceImageNotFound
except ImportError:
from glanceclient.v1.apiclient.exceptions import NotFound as GlanceImageNotFound
def __init__(self, nova_client, config):
class Compute(object):
def __init__(self, nova_client, neutron, config):
self.novaclient = nova_client
self.neutron = neutron
self.config = config
def find_image(self, image_name):
def find_image(self, glance_client, image_name):
try:
image = self.novaclient.images.find(name=image_name)
return image
except novaclient.exceptions.NotFound:
return None
return next(glance_client.images.list(filters={'name': image_name}), None)
except (novaclient.exceptions.NotFound, keystoneauth1.exceptions.http.NotFound,
GlanceImageNotFound):
pass
return None
def upload_image_via_url(self, glance_client, final_image_name,
image_url, retry_count=60):
@ -42,25 +50,22 @@ class Compute(object):
Directly uploads image to Nova via URL if image is not present
'''
retry = 0
file_prefix = "file://"
if not image_url.startswith(file_prefix):
LOG.error("File format %s is not supported. It must start with %s", image_url,
file_prefix)
return False
try:
# check image is file/url based.
file_prefix = "file://"
if image_url.startswith(file_prefix):
image_location = image_url.split(file_prefix)[1]
with open(image_location) as f_image:
img = glance_client.images.create(
name=str(final_image_name), disk_format="qcow2",
container_format="bare", is_public=True, data=f_image)
else:
# Upload the image
img = glance_client.images.create(
name=str(final_image_name), disk_format="qcow2",
container_format="bare", is_public=True,
copy_from=image_url)
image_location = image_url.split(file_prefix)[1]
f_image = open(image_location, 'rb')
img = glance_client.images.create(name=str(final_image_name))
glance_client.images.update(img.id, disk_format="qcow2",
container_format="bare")
glance_client.images.upload(img.id, f_image)
# Check for the image in glance
while img.status in ['queued', 'saving'] and retry < retry_count:
img = glance_client.images.find(name=img.name)
img = self.find_image(glance_client, image_name=img.name)
retry = retry + 1
LOG.debug("Image not yet active, retrying %s of %s...", retry, retry_count)
time.sleep(2)
@ -70,21 +75,17 @@ class Compute(object):
LOG.error("Cannot upload image without admin access. Please make "
"sure the image is uploaded and is either public or owned by you.")
return False
except IOError:
except (Exception, IOError):
# catch the exception for file based errors.
LOG.error("Failed while uploading the image. Please make sure the "
"image at the specified location %s is correct.", image_url)
return False
except Exception:
LOG.error("Failed while uploading the image, please make sure the "
"cloud under test has the access to URL: %s.", image_url)
return False
return True
def delete_image(self, glance_client, img_name):
try:
LOG.log("Deleting image %s...", img_name)
img = glance_client.images.find(name=img_name)
img = self.find_image(glance_client, img_name)
glance_client.images.delete(img.id)
except Exception:
LOG.error("Failed to delete the image %s.", img_name)
@ -150,7 +151,7 @@ class Compute(object):
config_drive=None, files=None, retry_count=10):
if sec_group:
security_groups = [sec_group.name]
security_groups = [sec_group["name"]]
else:
security_groups = None
@ -300,7 +301,7 @@ class Compute(object):
if hyp.zone == zone:
# matches
return az_host
# else continue - another zone with same host name?
# else continue - another zone with same host name?
# no match
LOG.error('No match for availability zone and host ' + az_host)
return None
@ -401,72 +402,85 @@ class Compute(object):
# Create a new security group with appropriate rules
def security_group_create(self):
# check first the security group exists
# May throw exceptions.NoUniqueMatch or NotFound
try:
group = self.novaclient.security_groups.find(name=self.config.security_group_name)
return group
except exceptions.NotFound:
group = self.novaclient.security_groups.create(name=self.config.security_group_name,
description="PNS Security group")
# Once security group try to find it iteratively
# (this check may no longer be necessary)
for _ in range(self.config.generic_retry_count):
group = self.novaclient.security_groups.get(group)
if group:
self.security_group_add_rules(group)
return group
else:
time.sleep(1)
return None
# except exceptions.NoUniqueMatch as exc:
# raise exc
for security_group in self.neutron.list_security_groups()["security_groups"]:
if security_group["name"] == self.config.security_group_name:
return security_group
group = self.neutron.create_security_group({
"security_group": {
"name": self.config.security_group_name,
"description": "PNS Security groups"}})
self.security_group_add_rules(group["security_group"])
return group["security_group"]
# Delete a security group
def security_group_delete(self, group):
if group:
LOG.info("Deleting security group")
self.novaclient.security_groups.delete(group)
self.neutron.delete_security_group(group["id"])
# Add rules to the security group
def security_group_add_rules(self, group):
# Allow ping traffic
self.novaclient.security_group_rules.create(group.id,
ip_protocol="icmp",
from_port=-1,
to_port=-1)
self.neutron.create_security_group_rule(
self.generate_security_group_rule_dict(group_id=group["id"],
protocol="icmp"))
if self.config.ipv6_mode:
self.novaclient.security_group_rules.create(group.id,
ip_protocol="icmp",
from_port=-1,
to_port=-1,
cidr="::/0")
self.neutron.create_security_group_rule(
self.generate_security_group_rule_dict(group_id=group["id"],
protocol="icmp",
ether_type="IPv6"))
# Allow SSH traffic
self.novaclient.security_group_rules.create(group.id,
ip_protocol="tcp",
from_port=22,
to_port=22)
self.neutron.create_security_group_rule(
self.generate_security_group_rule_dict(group_id=group["id"],
protocol="tcp",
port_range_min=22,
port_range_max=22))
# Allow TCP/UDP traffic for perf tools like iperf/nuttcp
# 5001: Data traffic (standard iperf data port)
# 5002: Control traffic (non standard)
# note that 5000/tcp is already picked by openstack keystone
if not self.config.ipv6_mode:
self.novaclient.security_group_rules.create(group.id,
ip_protocol="tcp",
from_port=5001,
to_port=5002)
self.novaclient.security_group_rules.create(group.id,
ip_protocol="udp",
from_port=5001,
to_port=5001)
self.neutron.create_security_group_rule(
self.generate_security_group_rule_dict(group_id=group["id"],
protocol="tcp",
port_range_min=5001,
port_range_max=5002))
self.neutron.create_security_group_rule(
self.generate_security_group_rule_dict(group_id=group["id"],
protocol="udp",
port_range_min=5001,
port_range_max=5002))
else:
# IPV6 rules addition
self.novaclient.security_group_rules.create(group.id,
ip_protocol="tcp",
from_port=5001,
to_port=5002,
cidr="::/0")
self.novaclient.security_group_rules.create(group.id,
ip_protocol="udp",
from_port=5001,
to_port=5001,
cidr="::/0")
self.neutron.create_security_group_rule(
self.generate_security_group_rule_dict(group_id=group["id"],
protocol="tcp",
ethertype="IPv6",
port_range_min=5001,
port_range_max=5002))
self.neutron.create_security_group_rule(
self.generate_security_group_rule_dict(group_id=group["id"],
protocol="udp",
ethertype="IPv6",
port_range_min=5001,
port_range_max=5002))
# Generates dict for security group rule
def generate_security_group_rule_dict(self, group_id, protocol, ethertype='IPv4',
port_range_min=None,
port_range_max=None):
return {
'security_group_rule': {
'direction': 'ingress',
'security_group_id': group_id,
'ethertype': ethertype,
'port_range_min': port_range_min,
'port_range_max': port_range_max,
'protocol': protocol,
'remote_group_id': None,
'remote_ip_prefix': '0.0.0.0/0'}}

35
vmtp/vmtp.py
View File

@ -30,7 +30,7 @@ import compute
from config import config_load
from config import config_loads
import credentials
from glanceclient import client as glanceclient
from glanceclient.v2 import client as glanceclient
import iperf_tool
from keystoneclient.auth.identity import v2 as keystone_v2
from keystoneclient.auth.identity import v3 as keystone_v3
@ -52,6 +52,8 @@ import sshutils
flow_num = 0
return_code = 0
class FlowPrinter(object):
@staticmethod
def print_desc(desc):
@ -60,8 +62,8 @@ class FlowPrinter(object):
CONLOG.info("=" * 60)
LOG.info('Flow %d: %s', flow_num, desc)
class ResultsCollector(object):
class ResultsCollector(object):
def __init__(self):
self.results = {'flows': []}
self.results['date'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
@ -109,7 +111,7 @@ class ResultsCollector(object):
def save_to_db(self, cfg):
'''Save results to MongoDB database.'''
LOG.info("Saving results to MongoDB database...")
post_id = pns_mongo.\
post_id = pns_mongo. \
pns_add_test_result_to_mongod(cfg.vmtp_mongod_ip,
cfg.vmtp_mongod_port,
cfg.vmtp_db,
@ -118,9 +120,11 @@ class ResultsCollector(object):
if post_id is None:
LOG.error("Failed to add result to DB")
class VmtpException(Exception):
pass
class VmtpTest(object):
def __init__(self, config, cred, rescol):
'''
@ -209,23 +213,25 @@ class VmtpTest(object):
# Create the nova and neutron instances
nova_client = novaclient.Client('2', session=sess)
neutron = neutronclient.Client('2.0', session=sess)
self.glance_client = glanceclient.Client('2', session=sess)
self.comp = compute.Compute(nova_client, self.config)
self.comp = compute.Compute(nova_client, neutron, self.config)
# Add the appropriate public key to openstack
self.comp.init_key_pair(self.config.public_key_name, self.instance_access)
self.image_instance = self.comp.find_image(self.config.image_name)
self.image_instance = self.comp.find_image(self.glance_client, self.config.image_name)
if self.image_instance is None:
print(len(self.config.vm_image_url))
if self.config.vm_image_url != "":
LOG.info('%s: image for VM not found, trying to upload it ...',
self.config.image_name)
self.glance_client = glanceclient.Client('1', session=sess)
self.comp.upload_image_via_url(
self.glance_client,
self.config.image_name,
self.config.vm_image_url)
self.image_instance = self.comp.find_image(self.config.image_name)
self.image_instance = self.comp.find_image(self.glance_client,
self.config.image_name)
self.image_uploaded = True
else:
# Exit the pogram
@ -478,6 +484,7 @@ class VmtpTest(object):
else:
self.teardown()
def test_native_tp(nhosts, ifname, config):
FlowPrinter.print_desc('Native Host to Host throughput')
result_list = []
@ -532,6 +539,7 @@ def test_native_tp(nhosts, ifname, config):
return result_list
def get_controller_info(ssh_access, net, res_col, retry_count):
if not ssh_access:
return
@ -558,6 +566,7 @@ def get_controller_info(ssh_access, net, res_col, retry_count):
res_col.add_properties(res)
def gen_report_data(proto, result):
try:
if proto in ['TCP', 'UDP', 'Multicast', 'ICMP']:
@ -607,8 +616,8 @@ def gen_report_data(proto, result):
traceback.print_exc()
return retval
def print_report(results):
def print_report(results):
# In order to parse the results with less logic, we are encoding the results as below:
# Same Network = 0, Different Network = 1
# Fixed IP = 0, Floating IP = 1
@ -708,6 +717,7 @@ def print_report(results):
global return_code
return_code = 1
def normalize_paths(cfg):
'''
Normalize the various paths to config files, tools, ssh priv and pub key
@ -721,6 +731,7 @@ def normalize_paths(cfg):
if cfg.private_key_file:
cfg.private_key_file = os.path.expanduser(os.path.expanduser(cfg.private_key_file))
def get_ssh_access(opt_name, opt_value, config):
'''Allocate a HostSshAccess instance to the option value
Check that a password is provided or the key pair in the config file
@ -738,6 +749,7 @@ def get_ssh_access(opt_name, opt_value, config):
sys.exit(2)
return host_access
def parse_opts_from_cli():
parser = argparse.ArgumentParser()
parser.add_argument('-c', '--config', dest='config',
@ -947,6 +959,7 @@ def parse_opts_from_cli():
return parser.parse_known_args()[0]
def decode_size_list(argname, size_list):
try:
pkt_sizes = size_list.split(',')
@ -958,8 +971,8 @@ def decode_size_list(argname, size_list):
sys.exit(1)
return pkt_sizes
def merge_opts_to_configs(opts):
def merge_opts_to_configs(opts):
default_cfg_file = resource_string(__name__, "cfg.default.yaml")
# read the default configuration file and possibly an override config file
# the precedence order is as follows:
@ -1056,7 +1069,6 @@ def merge_opts_to_configs(opts):
sys.exit(1)
config.vm_bandwidth = int(val * (10 ** (ex_unit * 3)))
# the pkt size for TCP, UDP and ICMP
if opts.tcp_pkt_sizes:
config.tcp_pkt_sizes = decode_size_list('--tcpbuf', opts.tcp_pkt_sizes)
@ -1130,6 +1142,7 @@ def merge_opts_to_configs(opts):
return config
def run_vmtp(opts):
'''Run VMTP
:param opts: Parameters that to be passed to VMTP in type argparse.Namespace(). See:
@ -1214,11 +1227,13 @@ def run_vmtp(opts):
return rescol.results
def main():
opts = parse_opts_from_cli()
log.setup('vmtp', debug=opts.debug, logfile=opts.logfile)
run_vmtp(opts)
sys.exit(return_code)
if __name__ == '__main__':
main()