x/kloudbuster
Code Issues Proposed changes

Allow availability zone auto-fill

Browse Source 
Change-Id: I4fbe8089271e58220bcf3b2e41c95cabaabf41c1
This commit is contained in:
ahothan 2015-02-13 12:19:23 -08:00
parent e4e294e5f8
commit d0a728db38
6 changed files with 162 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
README.rst
View File

@ -57,7 +57,6 @@ For running VMTP Docker Image
Docker is installed. See `here <https://docs.docker.com/installation/#installation/>`_ for instructions.
Sample Results Output
---------------------

7
cfg.default.yaml
View File

@ -2,7 +2,7 @@
# VMTP default configuration file
#
# This configuration file is ALWAYS loaded by VMTP and should never be modified by users.
# To specify your own user-specific property values, always define them in a separate config file
# To specify your own property values, always define them in a separate config file
# and pass that file to the script using -c or --config <file>
# Property values in that config file will override the default values in the current file
#
@ -29,7 +29,10 @@ flavor_type: 'm1.small'
# If the zone selected contains more than 1 compute node, the script
# will determine inter-node and intra-node throughput. If it contains only
# 1 compute node, only intra-node troughput will be measured.
availability_zone: 'nova'
# If empty (default), VMTP will automatically pick the first 2 hosts
# that are compute nodes regardless of the availability zone
#availability_zone: 'nova'
availability_zone:
# DNS server IP addresses to use for the VM (list of 1 or more DNS servers)
# This default DNS server is available on the Internet,

137
compute.py
View File

@ -213,23 +213,136 @@ class Compute(object):
flavor = self.novaclient.flavors.find(name=flavor_type)
return flavor
#
# Return a list of hosts which are in a specific availability zone
# May fail per policy in that case return an empty list
def list_hypervisor(self, zone_info):
if self.config.hypervisors:
print 'Using hypervisors:' + ', '.join(self.config.hypervisors)
return self.config.hypervisors
def normalize_az_host(self, az, host):
if not az:
az = self.config.availability_zone
return az + ':' + host
def auto_fill_az(self, host_list, host):
'''
no az provided, if there is a host list we can auto-fill the az
else we use the configured az if available
else we return an error
'''
if host_list:
for hyp in host_list:
if hyp.host_name == host:
return self.normalize_az_host(hyp.zone, host)
# no match on host
print('Error: passed host name does not exist: ' + host)
return None
if self.config.availability_zone:
return self.normalize_az_host(None, host)
print('Error: --hypervisor passed without an az and no az configured')
return None
def sanitize_az_host(self, host_list, az_host):
'''
host_list: list of hosts as retrieved from openstack (can be empty)
az_host: either a host or a az:host string
if a host, will check host is in the list, find the corresponding az and
return az:host
if az:host is passed will check the host is in the list and az matches
if host_list is empty, will return the configured az if there is no
az passed
'''
if ':' in az_host:
# no host_list, return as is (no check)
if not host_list:
return az_host
# if there is a host_list, extract and verify the az and host
az_host_list = az_host.split(':')
zone = az_host_list[0]
host = az_host_list[1]
for hyp in host_list:
if hyp.host_name == host:
if hyp.zone == zone:
# matches
return az_host
# else continue - another zone with same host name?
# no match
print('Error: no match for availability zone and host ' + az_host)
return None
else:
return self.auto_fill_az(host_list, az_host)
#
# Return a list of 0, 1 or 2 az:host
#
# The list is computed as follows:
# The list of all hosts is retrieved first from openstack
# if this fails, checks and az auto-fill are disabled
#
# If the user provides a list of hypervisors (--hypervisor)
# that list is checked and returned
#
# If the user provides a configured az name (config.availability_zone)
# up to the first 2 hosts from the list that match the az are returned
#
# If the user did not configure an az name
# up to the first 2 hosts from the list are returned
# Possible return values:
# [ az ]
# [ az:hyp ]
# [ az1:hyp1, az2:hyp2 ]
# [] if an error occurred (error message printed to console)
#
def get_az_host_list(self):
avail_list = []
host_list = []
try:
host_list = self.novaclient.hosts.list()
for host in host_list:
if host.zone == zone_info:
avail_list.append(host.host_name)
except novaclient.exceptions.Forbidden:
print ('Operation Forbidden: could not retrieve list of servers'
' in AZ (likely no permission)')
print ('Warning: Operation Forbidden: could not retrieve list of hosts'
' (likely no permission)')
# the user has specified a list of 1 or 2 hypervisors to use
if self.config.hypervisors:
for hyp in self.config.hypervisors:
hyp = self.sanitize_az_host(host_list, hyp)
if hyp:
avail_list.append(hyp)
else:
return []
# if the user did not specify an az, insert the configured az
if ':' not in hyp:
if self.config.availability_zone:
hyp = self.normalize_az_host(None, hyp)
else:
return []
# pick first 2 matches at most
if len(avail_list) == 2:
break
print 'Using hypervisors:' + ', '.join(avail_list)
else:
for host in host_list:
# this host must be a compute node
if host._info['service'] != 'compute':
continue
candidate = None
if self.config.availability_zone:
if host.zone == self.config.availability_zone:
candidate = self.normalize_az_host(None, host.host_name)
else:
candidate = self.normalize_az_host(host.zone, host.host_name)
if candidate:
avail_list.append(candidate)
# pick first 2 matches at most
if len(avail_list) == 2:
break
# if empty we insert the configured az
if not avail_list:
if not self.config.availability_zone:
print('Error: availability_zone must be configured')
elif host_list:
print('Error: no host matching the selection for availability zone: '
+ self.config.availability_zone)
avail_list = []
else:
avail_list = [self.config.availability_zone]
return avail_list
# Given 2 VMs test if they are running on same Host or not

20
doc/source/usage.rst
View File

@ -153,18 +153,26 @@ The default behavior for both TCP/UDP are unlimited. For TCP, we are leveraging
This is useful when running vmtp on production clouds. The test tool will use up all the bandwidth that may be needed by any other live VMs if we don't set any bandwidth limit. This feature will help to prevent impacting other VMs while running the test tool.
Host Selection in Availability Zone
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Host Selection and Availability Zone
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
The *--hypervisor* argument can be used to specify explicitly where to run the test VM in the configured availability zone.
VMTP requires 1 physical host to perform intra-node tests and 2 hosts to perform inter-node tests.
There are multiple ways to specify the placement of test VMs to VMTP. By default, VMTP will pick the first 2 compute hosts it can find, regardless of the availability zone.
This can be handy for example when exact VM placement can impact the data path performance (for example rack based placement when the availability zone spans across multiple racks).
It is possible to limit the host selection to a specific availability zone by specifying its name in the yaml configuration file ('availability_name' parameter).
The *--hypervisor* argument can also be used to specify explicitly on which hosts to run the test VMs.
The first *--hypervisor* argument specifies on which host to run the test server VM. The second *--hypervisor* argument (in the command line) specifies on which host to run the test client VMs.
The syntax to use for the argument value is either availability_zone and host name separated by a column (e.g. "--hypervisor nova:host26") or host name (e.g. "--hypervisor host12").
In the latter case, VMTP will automaticaly pick the availability zone of the host.
The value of the argument must match the hypervisor host name as known by OpenStack (or as displayed using "nova hypervisor-list")
Picking a particular host can be handy for example when exact VM placement can impact the data path performance (for example rack based placement).
The first --hypervisor argument specifies on which host to run the test server VM. The second --hypervisor argument specifies on which host to run the test client VMs.
The value of the argument must match the hypervisor host name as known by OpenStack (or as displayed using "nova hypervisor-list").
If an availability zone is provided, VMTP will check that the host name exists in that availability zone.
Example of usage is given below.
Upload Images to Glance
^^^^^^^^^^^^^^^^^^^^^^^

3
requirements.txt
View File

@ -7,11 +7,14 @@ Babel>=1.3
configure>=0.5
ecdsa>=0.11
jsonpatch>=1.9
jsonschema>=2.4.0
lxml>=3.4.0
oslo.utils>=1.2.0
paramiko>=1.14.0
pycrypto>=2.6.1
pymongo>=2.7.2
python-glanceclient>=0.15.0
python-neutronclient<3,>=2.3.6
python-novaclient>=2.18.1
python-openstackclient>=0.4.1

37
vmtp.py
View File

@ -261,28 +261,21 @@ class VmtpTest(object):
self.assert_true(self.net.vm_int_net)
# Get hosts for the availability zone to use
avail_list = self.comp.list_hypervisor(config.availability_zone)
# avail_list = self.comp.list_hypervisor(config.availability_zone)
avail_list = self.comp.get_az_host_list()
if not avail_list:
sys.exit(5)
# compute the list of client vm placements to run
if avail_list:
server_az = config.availability_zone + ":" + avail_list[0]
if len(avail_list) > 1:
# can do intra + inter
if config.inter_node_only:
# inter-node only
self.client_az_list = [config.availability_zone +
":" + avail_list[1]]
else:
self.client_az_list = [server_az, config.availability_zone +
":" + avail_list[1]]
else:
# can only do intra
self.client_az_list = [server_az]
else:
# cannot get the list of hosts
# can do intra or inter (cannot know)
server_az = config.availability_zone
self.client_az_list = [server_az]
# the first host is always where the server runs
server_az = avail_list[0]
if len(avail_list) > 1:
# 2 hosts are known
if config.inter_node_only:
# in this case we do not want the client to run on the same host
# as the server
avail_list.pop(0)
self.client_az_list = avail_list
self.server = PerfInstance(config.vm_name_server,
config,
@ -537,8 +530,8 @@ if __name__ == '__main__':
parser.add_argument('--hypervisor', dest='hypervisors',
action='append',
help='hypervisor to use in the avail zone (1 per arg, up to 2 args)',
metavar='name')
help='hypervisor to use (1 per arg, up to 2 args)',
metavar='[az:]hostname')
parser.add_argument('--inter-node-only', dest='inter_node_only',
default=False,