x/vmtp
Code Issues Proposed changes

add log for logstash

Browse Source 
Change-Id: Ifef5d27e702a13bab7817900f3870441a13732d6
This commit is contained in:
Xin 2016-06-01 18:29:14 -07:00
parent 5fbc516b2c
commit c1c919ae33
12 changed files with 123 additions and 151 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
vmtp/compute.py
View File

@ -15,18 +15,14 @@
'''Module for Openstack compute operations''' '''Module for Openstack compute operations'''
import log
import os import os
import time import time
import glanceclient.exc as glance_exception import glanceclient.exc as glance_exception
from log import LOG
import novaclient import novaclient
import novaclient.exceptions as exceptions import novaclient.exceptions as exceptions
CONLOG = log.getLogger('vmtp', 'console')
LSLOG = log.getLogger('vmtp', 'logstash')
LOG = log.getLogger('vmtp', 'all')
class Compute(object): class Compute(object):
def __init__(self, nova_client, config): def __init__(self, nova_client, config):
@ -66,32 +62,32 @@ class Compute(object):
while img.status in ['queued', 'saving'] and retry < retry_count: while img.status in ['queued', 'saving'] and retry < retry_count:
img = glance_client.images.find(name=img.name) img = glance_client.images.find(name=img.name)
retry = retry + 1 retry = retry + 1
CONLOG.debug("Image not yet active, retrying %s of %s..." % (retry, retry_count)) LOG.debug("Image not yet active, retrying %s of %s..." % (retry, retry_count))
time.sleep(2) time.sleep(2)
if img.status != 'active': if img.status != 'active':
raise Exception raise Exception
except glance_exception.HTTPForbidden: except glance_exception.HTTPForbidden:
CONLOG.error("Cannot upload image without admin access. Please make " LOG.error("Cannot upload image without admin access. Please make "
"sure the image is uploaded and is either public or owned by you.") "sure the image is uploaded and is either public or owned by you.")
return False return False
except IOError: except IOError:
# catch the exception for file based errors. # catch the exception for file based errors.
CONLOG.error("Failed while uploading the image. Please make sure the " LOG.error("Failed while uploading the image. Please make sure the "
"image at the specified location %s is correct." % image_url) "image at the specified location %s is correct." % image_url)
return False return False
except Exception: except Exception:
CONLOG.error("Failed while uploading the image, please make sure the " LOG.error("Failed while uploading the image, please make sure the "
"cloud under test has the access to URL: %s." % image_url) "cloud under test has the access to URL: %s." % image_url)
return False return False
return True return True
def delete_image(self, glance_client, img_name): def delete_image(self, glance_client, img_name):
try: try:
CONLOG.log("Deleting image %s..." % img_name) LOG.log("Deleting image %s..." % img_name)
img = glance_client.images.find(name=img_name) img = glance_client.images.find(name=img_name)
glance_client.images.delete(img.id) glance_client.images.delete(img.id)
except Exception: except Exception:
CONLOG.error("Failed to delete the image %s." % img_name) LOG.error("Failed to delete the image %s." % img_name)
return False return False
return True return True
@ -102,7 +98,7 @@ class Compute(object):
for key in keypair_list: for key in keypair_list:
if key.name == name: if key.name == name:
self.novaclient.keypairs.delete(name) self.novaclient.keypairs.delete(name)
CONLOG.info('Removed public key %s' % (name)) LOG.info('Removed public key %s' % (name))
break break
# Test if keypair file is present if not create it # Test if keypair file is present if not create it
@ -126,7 +122,7 @@ class Compute(object):
with open(os.path.expanduser(public_key_file)) as pkf: with open(os.path.expanduser(public_key_file)) as pkf:
public_key = pkf.read() public_key = pkf.read()
except IOError as exc: except IOError as exc:
CONLOG.error('Cannot open public key file %s: %s' % (public_key_file, exc)) LOG.error('Cannot open public key file %s: %s' % (public_key_file, exc))
return None return None
keypair = self.novaclient.keypairs.create(name, public_key) keypair = self.novaclient.keypairs.create(name, public_key)
return keypair return keypair
@ -176,14 +172,14 @@ class Compute(object):
if instance.status == 'ACTIVE': if instance.status == 'ACTIVE':
return instance return instance
if instance.status == 'ERROR': if instance.status == 'ERROR':
CONLOG.error('Instance creation error:' + instance.fault['message']) LOG.error('Instance creation error:' + instance.fault['message'])
break break
CONLOG.debug("[%s] VM status=%s, retrying %s of %s..." % LOG.debug("[%s] VM status=%s, retrying %s of %s..." %
(vmname, instance.status, (retry_attempt + 1), retry_count)) (vmname, instance.status, (retry_attempt + 1), retry_count))
time.sleep(2) time.sleep(2)
# instance not in ACTIVE state # instance not in ACTIVE state
CONLOG.error('Instance failed status=' + instance.status) LOG.error('Instance failed status=' + instance.status)
self.delete_server(instance) self.delete_server(instance)
return None return None
@ -213,10 +209,10 @@ class Compute(object):
if server.name == vmname and server.status == "ACTIVE": if server.name == vmname and server.status == "ACTIVE":
return True return True
# Sleep between retries # Sleep between retries
CONLOG.debug("[%s] VM not yet found, retrying %s of %s..." % LOG.debug("[%s] VM not yet found, retrying %s of %s..." %
(vmname, (retry_attempt + 1), retry_count)) (vmname, (retry_attempt + 1), retry_count))
time.sleep(2) time.sleep(2)
CONLOG.error("[%s] VM not found, after %s attempts" % (vmname, retry_count)) LOG.error("[%s] VM not found, after %s attempts" % (vmname, retry_count))
return False return False
# Returns True if server is found and deleted/False if not, # Returns True if server is found and deleted/False if not,
@ -225,7 +221,7 @@ class Compute(object):
servers_list = self.get_server_list() servers_list = self.get_server_list()
for server in servers_list: for server in servers_list:
if server.name == vmname: if server.name == vmname:
CONLOG.info('Deleting server %s' % (server)) LOG.info('Deleting server %s' % (server))
self.novaclient.servers.delete(server) self.novaclient.servers.delete(server)
return True return True
return False return False
@ -253,11 +249,11 @@ class Compute(object):
if hyp.host == host: if hyp.host == host:
return self.normalize_az_host(hyp.zone, host) return self.normalize_az_host(hyp.zone, host)
# no match on host # no match on host
CONLOG.error('Passed host name does not exist: ' + host) LOG.error('Passed host name does not exist: ' + host)
return None return None
if self.config.availability_zone: if self.config.availability_zone:
return self.normalize_az_host(None, host) return self.normalize_az_host(None, host)
CONLOG.error('--hypervisor passed without an az and no az configured') LOG.error('--hypervisor passed without an az and no az configured')
return None return None
def sanitize_az_host(self, host_list, az_host): def sanitize_az_host(self, host_list, az_host):
@ -285,7 +281,7 @@ class Compute(object):
return az_host return az_host
# else continue - another zone with same host name? # else continue - another zone with same host name?
# no match # no match
CONLOG.error('No match for availability zone and host ' + az_host) LOG.error('No match for availability zone and host ' + az_host)
return None return None
else: else:
return self.auto_fill_az(host_list, az_host) return self.auto_fill_az(host_list, az_host)
@ -318,8 +314,8 @@ class Compute(object):
try: try:
host_list = self.novaclient.services.list() host_list = self.novaclient.services.list()
except novaclient.exceptions.Forbidden: except novaclient.exceptions.Forbidden:
CONLOG.warning('Operation Forbidden: could not retrieve list of hosts' LOG.warning('Operation Forbidden: could not retrieve list of hosts'
' (likely no permission)') ' (likely no permission)')
# the user has specified a list of 1 or 2 hypervisors to use # the user has specified a list of 1 or 2 hypervisors to use
if self.config.hypervisors: if self.config.hypervisors:
@ -338,7 +334,7 @@ class Compute(object):
# pick first 2 matches at most # pick first 2 matches at most
if len(avail_list) == 2: if len(avail_list) == 2:
break break
CONLOG.info('Using hypervisors ' + ', '.join(avail_list)) LOG.info('Using hypervisors ' + ', '.join(avail_list))
else: else:
for host in host_list: for host in host_list:
# this host must be a compute node # this host must be a compute node
@ -360,10 +356,10 @@ class Compute(object):
if not avail_list: if not avail_list:
if not self.config.availability_zone: if not self.config.availability_zone:
CONLOG.error('Availability_zone must be configured') LOG.error('Availability_zone must be configured')
elif host_list: elif host_list:
CONLOG.error('No host matching the selection for availability zone: ' + LOG.error('No host matching the selection for availability zone: ' +
self.config.availability_zone) self.config.availability_zone)
avail_list = [] avail_list = []
else: else:
avail_list = [self.config.availability_zone] avail_list = [self.config.availability_zone]
@ -379,7 +375,7 @@ class Compute(object):
else: else:
return False return False
except novaclient.exceptions: except novaclient.exceptions:
CONLOG.warning("Exception in retrieving the hostId of servers") LOG.warning("Exception in retrieving the hostId of servers")
# Create a new security group with appropriate rules # Create a new security group with appropriate rules
def security_group_create(self): def security_group_create(self):
@ -407,7 +403,7 @@ class Compute(object):
# Delete a security group # Delete a security group
def security_group_delete(self, group): def security_group_delete(self, group):
if group: if group:
CONLOG.info("Deleting security group") LOG.info("Deleting security group")
self.novaclient.security_groups.delete(group) self.novaclient.security_groups.delete(group)
# Add rules to the security group # Add rules to the security group

5
vmtp/config.py
View File

@ -14,13 +14,8 @@
# #
from attrdict import AttrDict from attrdict import AttrDict
import log
import yaml import yaml
CONLOG = log.getLogger('vmtp', 'console')
LSLOG = log.getLogger('vmtp', 'logstash')
LOG = log.getLogger('vmtp', 'all')
def config_load(file_name, from_cfg=None): def config_load(file_name, from_cfg=None):
'''Load a yaml file into a config dict, merge with from_cfg if not None '''Load a yaml file into a config dict, merge with from_cfg if not None
The config file content taking precedence in case of duplicate The config file content taking precedence in case of duplicate

9
vmtp/credentials.py
View File

@ -15,13 +15,10 @@
# Module for credentials in Openstack # Module for credentials in Openstack
import getpass import getpass
import log
import os import os
import re import re
CONLOG = log.getLogger('vmtp', 'console') from log import LOG
LSLOG = log.getLogger('vmtp', 'logstash')
LOG = log.getLogger('vmtp', 'all')
class Credentials(object): class Credentials(object):
@ -90,7 +87,7 @@ class Credentials(object):
elif name == "CACERT": elif name == "CACERT":
self.rc_cacert = value self.rc_cacert = value
else: else:
CONLOG.error('Error: rc file does not exist %s' % (openrc_file)) LOG.error('Error: rc file does not exist %s' % (openrc_file))
success = False success = False
elif not no_env: elif not no_env:
# no openrc file passed - we assume the variables have been # no openrc file passed - we assume the variables have been
@ -98,7 +95,7 @@ class Credentials(object):
# just check that they are present # just check that they are present
for varname in ['OS_USERNAME', 'OS_AUTH_URL', 'OS_TENANT_NAME']: for varname in ['OS_USERNAME', 'OS_AUTH_URL', 'OS_TENANT_NAME']:
if varname not in os.environ: if varname not in os.environ:
CONLOG.warning('%s is missing' % (varname)) LOG.warning('%s is missing' % (varname))
success = False success = False
if success: if success:
self.rc_username = os.environ['OS_USERNAME'] self.rc_username = os.environ['OS_USERNAME']

9
vmtp/instance.py
View File

@ -15,14 +15,11 @@
import re import re
import log from log import LOG
import monitor import monitor
from netaddr import IPAddress from netaddr import IPAddress
import sshutils import sshutils
CONLOG = log.getLogger('vmtp', 'console')
LSLOG = log.getLogger('vmtp', 'logstash')
LOG = log.getLogger('vmtp', 'all')
# a dictionary of sequence number indexed by a name prefix # a dictionary of sequence number indexed by a name prefix
prefix_seq = {} prefix_seq = {}
@ -178,11 +175,11 @@ class Instance(object):
# Display a status message with the standard header that has the instance # Display a status message with the standard header that has the instance
# name (e.g. [foo] some text) # name (e.g. [foo] some text)
def display(self, fmt, *args): def display(self, fmt, *args):
CONLOG.info(('[%s] ' + fmt) % ((self.name,) + args)) LOG.info(('[%s] ' + fmt) % ((self.name,) + args))
# Debugging message, to be printed only in debug mode # Debugging message, to be printed only in debug mode
def buginf(self, fmt, *args): def buginf(self, fmt, *args):
CONLOG.debug(('[%s] ' + fmt) % ((self.name,) + args)) LOG.debug(('[%s] ' + fmt) % ((self.name,) + args))
# Ping an IP from this instance # Ping an IP from this instance
def ping_check(self, target_ip, ping_count, pass_threshold): def ping_check(self, target_ip, ping_count, pass_threshold):

8
vmtp/iperf_tool.py
View File

@ -15,13 +15,9 @@
import re import re
import log from log import LOG
from perf_tool import PerfTool from perf_tool import PerfTool
CONLOG = log.getLogger('vmtp', 'console')
LSLOG = log.getLogger('vmtp', 'logstash')
LOG = log.getLogger('vmtp', 'all')
# The resulting unit should be in K # The resulting unit should be in K
MULTIPLIERS = {'K': 1, MULTIPLIERS = {'K': 1,
'M': 1.0e3, 'M': 1.0e3,
@ -33,7 +29,7 @@ def get_bdw_kbps(bdw, bdw_unit):
return bdw / 1000 return bdw / 1000
if bdw_unit in MULTIPLIERS: if bdw_unit in MULTIPLIERS:
return int(bdw * MULTIPLIERS[bdw_unit]) return int(bdw * MULTIPLIERS[bdw_unit])
CONLOG.error('Error: unknown multiplier: ' + bdw_unit) LOG.error('Error: unknown multiplier: ' + bdw_unit)
return bdw return bdw
class IperfTool(PerfTool): class IperfTool(PerfTool):

10
vmtp/log.py
View File

@ -34,18 +34,22 @@ def setup(product_name, debug=False, logfile=None):
console_logger.addHandler(console_handler) console_logger.addHandler(console_handler)
console_logger.setLevel(log_level) console_logger.setLevel(log_level)
logstash_logger = logging.getLogger(product_name + '_' + 'logstash') file_logger = logging.getLogger(product_name + '_' + 'file')
logstash_logger.setLevel(log_level) file_logger.setLevel(log_level)
all_logger = logging.getLogger(product_name + '_' + 'all') all_logger = logging.getLogger(product_name + '_' + 'all')
all_logger.addHandler(console_handler) all_logger.addHandler(console_handler)
all_logger.setLevel(log_level) all_logger.setLevel(log_level)
if file_handler: if file_handler:
logstash_logger.addHandler(file_handler) file_logger.addHandler(file_handler)
all_logger.addHandler(file_handler) all_logger.addHandler(file_handler)
def getLogger(product, target): def getLogger(product, target):
logger = logging.getLogger(product + "_" + target) logger = logging.getLogger(product + "_" + target)
return logger return logger
CONLOG = getLogger('vmtp', 'console')
LOG = getLogger('vmtp', 'all')
FILELOG = getLogger('vmtp', 'file')

34
vmtp/network.py
View File

@ -15,16 +15,12 @@
import time import time
import log from log import LOG
# Module containing a helper class for operating on OpenStack networks # Module containing a helper class for operating on OpenStack networks
from neutronclient.common.exceptions import NetworkInUseClient from neutronclient.common.exceptions import NetworkInUseClient
from neutronclient.common.exceptions import NeutronException from neutronclient.common.exceptions import NeutronException
import vmtp import vmtp
CONLOG = log.getLogger('vmtp', 'console')
LSLOG = log.getLogger('vmtp', 'logstash')
LOG = log.getLogger('vmtp', 'all')
class Network(object): class Network(object):
# #
@ -80,10 +76,10 @@ class Network(object):
break break
if not self.ext_net: if not self.ext_net:
CONLOG.error("No external network found.") LOG.error("No external network found.")
return return
CONLOG.info("Using external network: " + self.ext_net['name']) LOG.info("Using external network: " + self.ext_net['name'])
# Find or create the router to the external network # Find or create the router to the external network
ext_net_id = self.ext_net['id'] ext_net_id = self.ext_net['id']
@ -93,7 +89,7 @@ class Network(object):
if external_gw_info: if external_gw_info:
if external_gw_info['network_id'] == ext_net_id: if external_gw_info['network_id'] == ext_net_id:
self.ext_router = router self.ext_router = router
CONLOG.info('Found external router: %s' % (self.ext_router['name'])) LOG.info('Found external router: %s' % (self.ext_router['name']))
break break
# create a new external router if none found and a name was given # create a new external router if none found and a name was given
@ -101,7 +97,7 @@ class Network(object):
if (not self.ext_router) and self.ext_router_name: if (not self.ext_router) and self.ext_router_name:
self.ext_router = self.create_router(self.ext_router_name, self.ext_router = self.create_router(self.ext_router_name,
self.ext_net['id']) self.ext_net['id'])
CONLOG.info('Created ext router %s.' % (self.ext_router_name)) LOG.info('Created ext router %s.' % (self.ext_router_name))
self.ext_router_created = True self.ext_router_created = True
if config.ipv6_mode: if config.ipv6_mode:
@ -146,7 +142,7 @@ class Network(object):
for network in self.networks: for network in self.networks:
if network['name'] == network_name: if network['name'] == network_name:
CONLOG.info('Found existing internal network: %s' % (network_name)) LOG.info('Found existing internal network: %s' % (network_name))
return network return network
body = { body = {
@ -185,7 +181,7 @@ class Network(object):
subnet = self.neutron_client.create_subnet(body)['subnet'] subnet = self.neutron_client.create_subnet(body)['subnet']
# add the subnet id to the network dict # add the subnet id to the network dict
network['subnets'].append(subnet['id']) network['subnets'].append(subnet['id'])
CONLOG.info('Created internal network: %s.' % (network_name)) LOG.info('Created internal network: %s.' % (network_name))
return network return network
# Delete a network and associated subnet # Delete a network and associated subnet
@ -196,7 +192,7 @@ class Network(object):
for _ in range(1, 5): for _ in range(1, 5):
try: try:
self.neutron_client.delete_network(network['id']) self.neutron_client.delete_network(network['id'])
CONLOG.info('Network %s deleted.' % (name)) LOG.info('Network %s deleted.' % (name))
break break
except NetworkInUseClient: except NetworkInUseClient:
time.sleep(1) time.sleep(1)
@ -220,7 +216,7 @@ class Network(object):
port_ip = port['fixed_ips'][0] port_ip = port['fixed_ips'][0]
if (port['device_id'] == self.ext_router['id']) and \ if (port['device_id'] == self.ext_router['id']) and \
(port_ip['subnet_id'] == self.vm_int_net[0]['subnets'][0]): (port_ip['subnet_id'] == self.vm_int_net[0]['subnets'][0]):
CONLOG.info('Ext router already associated to the internal network.') LOG.info('Ext router already associated to the internal network.')
return return
for int_net in self.vm_int_net: for int_net in self.vm_int_net:
@ -228,7 +224,7 @@ class Network(object):
'subnet_id': int_net['subnets'][0] 'subnet_id': int_net['subnets'][0]
} }
self.neutron_client.add_interface_router(self.ext_router['id'], body) self.neutron_client.add_interface_router(self.ext_router['id'], body)
CONLOG.debug('Ext router associated to ' + int_net['name']) LOG.debug('Ext router associated to ' + int_net['name'])
# If ipv6 is enabled than add second subnet # If ipv6 is enabled than add second subnet
if self.ipv6_enabled: if self.ipv6_enabled:
body = { body = {
@ -256,7 +252,7 @@ class Network(object):
except NeutronException: except NeutronException:
# May fail with neutronclient.common.exceptions.Conflict # May fail with neutronclient.common.exceptions.Conflict
# if there are floating IP in use - just ignore # if there are floating IP in use - just ignore
CONLOG.warning('Router interface may have floating IP in use: not deleted') LOG.warning('Router interface may have floating IP in use: not deleted')
# Lookup network given network name # Lookup network given network name
def lookup_network(self, network_name): def lookup_network(self, network_name):
@ -308,11 +304,11 @@ class Network(object):
body['port']['binding:vnic_type'] = vnic_type body['port']['binding:vnic_type'] = vnic_type
port = self.neutron_client.create_port(body) port = self.neutron_client.create_port(body)
if self.config.debug: if self.config.debug:
CONLOG.info('Created port ' + port['port']['id']) LOG.info('Created port ' + port['port']['id'])
return port['port'] return port['port']
def delete_port(self, port): def delete_port(self, port):
CONLOG.debug('Deleting port ' + port['id']) LOG.debug('Deleting port ' + port['id'])
self.neutron_client.delete_port(port['id']) self.neutron_client.delete_port(port['id'])
# Create a floating ip on the external network and return it # Create a floating ip on the external network and return it
@ -344,9 +340,9 @@ class Network(object):
self.neutron_client.remove_gateway_router( self.neutron_client.remove_gateway_router(
self.ext_router['id']) self.ext_router['id'])
self.neutron_client.delete_router(self.ext_router['id']) self.neutron_client.delete_router(self.ext_router['id'])
CONLOG.info('External router %s deleted' % (self.ext_router['name'])) LOG.info('External router %s deleted' % (self.ext_router['name']))
except TypeError: except TypeError:
CONLOG.info("No external router set") LOG.info("No external router set")
def _get_l2agent_type(self): def _get_l2agent_type(self):
''' '''

5
vmtp/nuttcp_tool.py
View File

@ -15,14 +15,9 @@
import re import re
import log
from perf_tool import PerfTool from perf_tool import PerfTool
import sshutils import sshutils
CONLOG = log.getLogger('vmtp', 'console')
LSLOG = log.getLogger('vmtp', 'logstash')
LOG = log.getLogger('vmtp', 'all')
class NuttcpTool(PerfTool): class NuttcpTool(PerfTool):
def __init__(self, instance): def __init__(self, instance):

5
vmtp/perf_instance.py
View File

@ -14,13 +14,8 @@
# #
from instance import Instance as Instance from instance import Instance as Instance
import log
from perf_tool import PingTool from perf_tool import PingTool
CONLOG = log.getLogger('vmtp', 'console')
LSLOG = log.getLogger('vmtp', 'logstash')
LOG = log.getLogger('vmtp', 'all')
class PerfInstance(Instance): class PerfInstance(Instance):
'''An openstack instance to run performance tools '''An openstack instance to run performance tools
''' '''

9
vmtp/perf_tool.py
View File

@ -16,16 +16,11 @@
import abc import abc
import re import re
import log
from pkg_resources import resource_filename from pkg_resources import resource_filename
# where to copy the tool on the target, must end with slash # where to copy the tool on the target, must end with slash
SCP_DEST_DIR = '/tmp/' SCP_DEST_DIR = '/tmp/'
CONLOG = log.getLogger('vmtp', 'console')
LSLOG = log.getLogger('vmtp', 'logstash')
LOG = log.getLogger('vmtp', 'all')
# #
# A base class for all tools that can be associated to an instance # A base class for all tools that can be associated to an instance
# #
@ -193,7 +188,7 @@ class PerfTool(object):
min_kbps = int((max_kbps + min_kbps) / 2) min_kbps = int((max_kbps + min_kbps) / 2)
kbps = int((max_kbps + min_kbps) / 2) kbps = int((max_kbps + min_kbps) / 2)
# CONLOG.debug(' undershot, min=%d kbps=%d max=%d' % (min_kbps, kbps, max_kbps)) # LOG.debug(' undershot, min=%d kbps=%d max=%d' % (min_kbps, kbps, max_kbps))
elif loss_rate > max_loss_rate: elif loss_rate > max_loss_rate:
# overshot # overshot
max_kbps = kbps max_kbps = kbps
@ -201,7 +196,7 @@ class PerfTool(object):
kbps = measured_kbps kbps = measured_kbps
else: else:
kbps = int((max_kbps + min_kbps) / 2) kbps = int((max_kbps + min_kbps) / 2)
# CONLOG.debug(' overshot, min=%d kbps=%d max=%d' % (min_kbps, kbps, max_kbps)) # LOG.debug(' overshot, min=%d kbps=%d max=%d' % (min_kbps, kbps, max_kbps))
else: else:
# converged within loss rate bracket # converged within loss rate bracket
break break

18
vmtp/sshutils.py
View File

@ -64,14 +64,10 @@ import StringIO
import sys import sys
import time import time
import log from log import LOG
import paramiko import paramiko
import scp import scp
CONLOG = log.getLogger('vmtp', 'console')
LSLOG = log.getLogger('vmtp', 'logstash')
LOG = log.getLogger('vmtp', 'all')
# from rally.openstack.common.gettextutils import _ # from rally.openstack.common.gettextutils import _
@ -412,7 +408,7 @@ class SSH(object):
if int(pkt_loss) < int(pass_threshold): if int(pkt_loss) < int(pass_threshold):
return 1 return 1
else: else:
CONLOG.error('Ping to %s failed: %s' % (target_ip, cmd_output)) LOG.error('Ping to %s failed: %s' % (target_ip, cmd_output))
return 0 return 0
def get_file_from_host(self, from_path, to_path): def get_file_from_host(self, from_path, to_path):
@ -425,7 +421,7 @@ class SSH(object):
try: try:
scpcon.get(from_path, to_path) scpcon.get(from_path, to_path)
except scp.SCPException as exp: except scp.SCPException as exp:
CONLOG.error("Receive failed: [%s]" % exp) LOG.error("Receive failed: [%s]" % exp)
return 0 return 0
return 1 return 1
@ -439,7 +435,7 @@ class SSH(object):
try: try:
scpcon.put(from_path, remote_path=to_path) scpcon.put(from_path, remote_path=to_path)
except scp.SCPException as exp: except scp.SCPException as exp:
CONLOG.error("Send failed: [%s]" % exp) LOG.error("Send failed: [%s]" % exp)
return 0 return 0
return 1 return 1
@ -465,7 +461,7 @@ class SSH(object):
if self.stat(os_release_file): if self.stat(os_release_file):
data = self.read_remote_file(os_release_file) data = self.read_remote_file(os_release_file)
if data is None: if data is None:
CONLOG.error("Failed to read file %s" % os_release_file) LOG.error("Failed to read file %s" % os_release_file)
return None return None
for line in data.splitlines(): for line in data.splitlines():
@ -483,7 +479,7 @@ class SSH(object):
if self.stat(sys_release_file): if self.stat(sys_release_file):
data = self.read_remote_file(sys_release_file) data = self.read_remote_file(sys_release_file)
if data is None: if data is None:
CONLOG.error("Failed to read file %s" % sys_release_file) LOG.error("Failed to read file %s" % sys_release_file)
return None return None
for line in data.splitlines(): for line in data.splitlines():
@ -512,7 +508,7 @@ class SSH(object):
if mobj: if mobj:
return mobj.group(0) return mobj.group(0)
CONLOG.info("%s pkg installed " % rpm_pkg) LOG.info("%s pkg installed " % rpm_pkg)
return None return None

96
vmtp/vmtp.py
View File

@ -33,6 +33,9 @@ import credentials
from glanceclient.v1 import client as glanceclient from glanceclient.v1 import client as glanceclient
import iperf_tool import iperf_tool
from keystoneclient.v2_0 import client as keystoneclient from keystoneclient.v2_0 import client as keystoneclient
from log import CONLOG
from log import FILELOG
from log import LOG
import network import network
from neutronclient.v2_0 import client as neutronclient from neutronclient.v2_0 import client as neutronclient
from novaclient.client import Client from novaclient.client import Client
@ -44,10 +47,6 @@ import pns_mongo
from prettytable import PrettyTable from prettytable import PrettyTable
import sshutils import sshutils
CONLOG = log.getLogger('vmtp', 'console')
LSLOG = log.getLogger('vmtp', 'logstash')
LOG = log.getLogger('vmtp', 'all')
flow_num = 0 flow_num = 0
class FlowPrinter(object): class FlowPrinter(object):
@staticmethod @staticmethod
@ -55,7 +54,7 @@ class FlowPrinter(object):
global flow_num global flow_num
flow_num = flow_num + 1 flow_num = flow_num + 1
CONLOG.info("=" * 60) CONLOG.info("=" * 60)
CONLOG.info('Flow %d: %s' % (flow_num, desc)) LOG.info('Flow %d: %s' % (flow_num, desc))
class ResultsCollector(object): class ResultsCollector(object):
@ -105,7 +104,7 @@ class ResultsCollector(object):
def save_to_db(self, cfg): def save_to_db(self, cfg):
'''Save results to MongoDB database.''' '''Save results to MongoDB database.'''
CONLOG.info("Saving results to MongoDB database...") LOG.info("Saving results to MongoDB database...")
post_id = pns_mongo.\ post_id = pns_mongo.\
pns_add_test_result_to_mongod(cfg.vmtp_mongod_ip, pns_add_test_result_to_mongod(cfg.vmtp_mongod_ip,
cfg.vmtp_mongod_port, cfg.vmtp_mongod_port,
@ -113,7 +112,7 @@ class ResultsCollector(object):
cfg.vmtp_collection, cfg.vmtp_collection,
self.results) self.results)
if post_id is None: if post_id is None:
CONLOG.error("Failed to add result to DB") LOG.error("Failed to add result to DB")
class VmtpException(Exception): class VmtpException(Exception):
pass pass
@ -184,14 +183,14 @@ class VmtpTest(object):
self.instance_access.public_key_file = pub_key self.instance_access.public_key_file = pub_key
self.instance_access.private_key_file = priv_key self.instance_access.private_key_file = priv_key
else: else:
CONLOG.error('Default keypair ~/.ssh/id_rsa[.pub] does not exist. Please ' LOG.error('Default keypair ~/.ssh/id_rsa[.pub] does not exist. Please '
'either create one in your home directory, or specify your ' 'either create one in your home directory, or specify your '
'keypair information in the config file before running VMTP.') 'keypair information in the config file before running VMTP.')
sys.exit(1) sys.exit(1)
if self.config.debug and self.instance_access.public_key_file: if self.config.debug and self.instance_access.public_key_file:
CONLOG.info('VM public key: ' + self.instance_access.public_key_file) LOG.info('VM public key: ' + self.instance_access.public_key_file)
CONLOG.info('VM private key: ' + self.instance_access.private_key_file) LOG.info('VM private key: ' + self.instance_access.private_key_file)
# If we need to reuse existing vms just return without setup # If we need to reuse existing vms just return without setup
if not self.config.reuse_existing_vm: if not self.config.reuse_existing_vm:
@ -209,8 +208,8 @@ class VmtpTest(object):
self.image_instance = self.comp.find_image(self.config.image_name) self.image_instance = self.comp.find_image(self.config.image_name)
if self.image_instance is None: if self.image_instance is None:
if self.config.vm_image_url != "": if self.config.vm_image_url != "":
CONLOG.info('%s: image for VM not found, trying to upload it ...' % LOG.info('%s: image for VM not found, trying to upload it ...' %
(self.config.image_name)) (self.config.image_name))
keystone = keystoneclient.Client(**creds) keystone = keystoneclient.Client(**creds)
glance_endpoint = keystone.service_catalog.url_for( glance_endpoint = keystone.service_catalog.url_for(
service_type='image', endpoint_type='publicURL') service_type='image', endpoint_type='publicURL')
@ -225,24 +224,24 @@ class VmtpTest(object):
self.image_uploaded = True self.image_uploaded = True
else: else:
# Exit the pogram # Exit the pogram
CONLOG.error('%s: image to launch VM not found. ABORTING.' % LOG.error('%s: image to launch VM not found. ABORTING.' %
(self.config.image_name)) (self.config.image_name))
sys.exit(1) sys.exit(1)
self.assert_true(self.image_instance) self.assert_true(self.image_instance)
CONLOG.info('Found image %s to launch VM, will continue' % (self.config.image_name)) LOG.info('Found image %s to launch VM, will continue' % (self.config.image_name))
self.flavor_type = self.comp.find_flavor(self.config.flavor_type) self.flavor_type = self.comp.find_flavor(self.config.flavor_type)
self.net = network.Network(neutron, self.config) self.net = network.Network(neutron, self.config)
self.rescol.add_property('l2agent_type', self.net.l2agent_type) self.rescol.add_property('l2agent_type', self.net.l2agent_type)
CONLOG.info("OpenStack agent: " + self.net.l2agent_type) LOG.info("OpenStack agent: " + self.net.l2agent_type)
try: try:
network_type = self.net.vm_int_net[0]['provider:network_type'] network_type = self.net.vm_int_net[0]['provider:network_type']
CONLOG.info("OpenStack network type: " + network_type) LOG.info("OpenStack network type: " + network_type)
self.rescol.add_property('encapsulation', network_type) self.rescol.add_property('encapsulation', network_type)
except KeyError as exp: except KeyError as exp:
network_type = 'Unknown' network_type = 'Unknown'
CONLOG.info("Provider network type not found: ", str(exp)) LOG.info("Provider network type not found: ", str(exp))
# Create a new security group for the test # Create a new security group for the test
self.sec_group = self.comp.security_group_create() self.sec_group = self.comp.security_group_create()
@ -321,6 +320,7 @@ class VmtpTest(object):
if res: if res:
self.rescol.add_flow_result(res) self.rescol.add_flow_result(res)
CONLOG.info(self.rescol.ppr.pformat(res)) CONLOG.info(self.rescol.ppr.pformat(res))
FILELOG.info(json.dumps(res, sort_keys=True))
client.dispose() client.dispose()
def add_location(self, label): def add_location(self, label):
@ -356,12 +356,14 @@ class VmtpTest(object):
results_list = perf_output['results'] results_list = perf_output['results']
for res_dict in results_list: for res_dict in results_list:
if 'error' in res_dict: if 'error' in res_dict:
CONLOG.error('Stopping execution on error, cleanup all VMs/networks manually') LOG.error('Stopping execution on error, cleanup all VMs/networks manually')
CONLOG.info(self.rescol.ppr.pformat(perf_output)) CONLOG.info(self.rescol.ppr.pformat(perf_output))
FILELOG.info(json.dumps(perf_output, sort_keys=True))
sys.exit(2) sys.exit(2)
self.rescol.add_flow_result(perf_output) self.rescol.add_flow_result(perf_output)
CONLOG.info(self.rescol.ppr.pformat(perf_output)) CONLOG.info(self.rescol.ppr.pformat(perf_output))
FILELOG.info(json.dumps(perf_output, sort_keys=True))
def measure_vm_flows(self): def measure_vm_flows(self):
# scenarios need to be tested for both inter and intra node # scenarios need to be tested for both inter and intra node
@ -397,7 +399,7 @@ class VmtpTest(object):
''' '''
Clean up the floating ip and VMs Clean up the floating ip and VMs
''' '''
CONLOG.info('Cleaning up...') LOG.info('Cleaning up...')
if self.server: if self.server:
self.server.dispose() self.server.dispose()
if self.client: if self.client:
@ -413,7 +415,7 @@ class VmtpTest(object):
self.comp.security_group_delete(self.sec_group) self.comp.security_group_delete(self.sec_group)
except ClientException: except ClientException:
# May throw novaclient.exceptions.BadRequest if in use # May throw novaclient.exceptions.BadRequest if in use
CONLOG.warning('Security group in use: not deleted') LOG.warning('Security group in use: not deleted')
if self.image_uploaded and self.config.delete_image_after_run: if self.image_uploaded and self.config.delete_image_after_run:
self.comp.delete_image(self.glance_client, self.config.image_name) self.comp.delete_image(self.glance_client, self.config.image_name)
@ -426,11 +428,11 @@ class VmtpTest(object):
except KeyboardInterrupt: except KeyboardInterrupt:
traceback.format_exc() traceback.format_exc()
except (VmtpException, sshutils.SSHError, ClientException, Exception): except (VmtpException, sshutils.SSHError, ClientException, Exception):
CONLOG.error(traceback.print_exc()) LOG.error(traceback.print_exc())
error_flag = True error_flag = True
if self.config.stop_on_error and error_flag: if self.config.stop_on_error and error_flag:
CONLOG.error('Stopping execution on error, cleanup all VMs/networks manually') LOG.error('Stopping execution on error, cleanup all VMs/networks manually')
sys.exit(2) sys.exit(2)
else: else:
self.teardown() self.teardown()
@ -458,7 +460,7 @@ def test_native_tp(nhosts, ifname, config):
# use the IP address configured on given interface # use the IP address configured on given interface
server_ip = server.get_interface_ip(ifname) server_ip = server.get_interface_ip(ifname)
if not server_ip: if not server_ip:
CONLOG.error('Cannot get IP address for interface ' + ifname) LOG.error('Cannot get IP address for interface ' + ifname)
else: else:
server.display('Clients will use server IP address %s (%s)' % server.display('Clients will use server IP address %s (%s)' %
(server_ip, ifname)) (server_ip, ifname))
@ -492,10 +494,10 @@ def test_native_tp(nhosts, ifname, config):
def get_controller_info(ssh_access, net, res_col, retry_count): def get_controller_info(ssh_access, net, res_col, retry_count):
if not ssh_access: if not ssh_access:
return return
CONLOG.info('Fetching OpenStack deployment details...') LOG.info('Fetching OpenStack deployment details...')
sshcon = sshutils.SSH(ssh_access, connect_retry_count=retry_count) sshcon = sshutils.SSH(ssh_access, connect_retry_count=retry_count)
if sshcon is None: if sshcon is None:
CONLOG.error('Cannot connect to the controller node') LOG.error('Cannot connect to the controller node')
return return
res = {} res = {}
res['distro'] = sshcon.get_host_os_version() res['distro'] = sshcon.get_host_os_version()
@ -511,6 +513,8 @@ def get_controller_info(ssh_access, net, res_col, retry_count):
res['l2agent_version'] = sshcon.get_l2agent_version(l2type) res['l2agent_version'] = sshcon.get_l2agent_version(l2type)
# print results # print results
CONLOG.info(res_col.ppr.pformat(res)) CONLOG.info(res_col.ppr.pformat(res))
FILELOG.info(json.dumps(res, sort_keys=True))
res_col.add_properties(res) res_col.add_properties(res)
def gen_report_data(proto, result): def gen_report_data(proto, result):
@ -639,6 +643,11 @@ def print_report(results):
summary += str(ptable) summary += str(ptable)
CONLOG.info(summary) CONLOG.info(summary)
ls_summary = {"Result": results, "Total Scenarios": (len(table) - 1),
"Passed Scenarios": "%d [%.2f%%]" % (cnt_passed, passed_rate),
"Failed Scenarios": "%d [%.2f%%]" % (cnt_failed, failed_rate),
"Skipped Scenarios": "%d" % (cnt_skipped)}
FILELOG.info(json.dumps(ls_summary, sort_keys=True))
def normalize_paths(cfg): def normalize_paths(cfg):
''' '''
@ -666,7 +675,7 @@ def get_ssh_access(opt_name, opt_value, config):
host_access.private_key_file = config.private_key_file host_access.private_key_file = config.private_key_file
host_access.public_key_file = config.public_key_file host_access.public_key_file = config.public_key_file
if host_access.error: if host_access.error:
CONLOG.error('Error for --' + (opt_name + ':' + host_access.error)) LOG.error('Error for --' + (opt_name + ':' + host_access.error))
sys.exit(2) sys.exit(2)
return host_access return host_access
@ -880,15 +889,15 @@ def merge_opts_to_configs(opts):
config.same_network_only = opts.same_network_only config.same_network_only = opts.same_network_only
if config.public_key_file and not os.path.isfile(config.public_key_file): if config.public_key_file and not os.path.isfile(config.public_key_file):
CONLOG.warning('Invalid public_key_file:' + config.public_key_file) LOG.warning('Invalid public_key_file:' + config.public_key_file)
config.public_key_file = None config.public_key_file = None
if config.private_key_file and not os.path.isfile(config.private_key_file): if config.private_key_file and not os.path.isfile(config.private_key_file):
CONLOG.warning('Invalid private_key_file:' + config.private_key_file) LOG.warning('Invalid private_key_file:' + config.private_key_file)
config.private_key_file = None config.private_key_file = None
# direct: use SR-IOV ports for all the test VMs # direct: use SR-IOV ports for all the test VMs
if opts.vnic_type not in [None, 'direct', 'macvtap', 'normal']: if opts.vnic_type not in [None, 'direct', 'macvtap', 'normal']:
CONLOG.error('Invalid vnic-type: ' + opts.vnic_type) LOG.error('Invalid vnic-type: ' + opts.vnic_type)
sys.exit(1) sys.exit(1)
config.vnic_type = opts.vnic_type config.vnic_type = opts.vnic_type
config.hypervisors = opts.hypervisors config.hypervisors = opts.hypervisors
@ -937,8 +946,8 @@ def merge_opts_to_configs(opts):
raise ValueError raise ValueError
val = int(opts.vm_bandwidth[0:-1]) val = int(opts.vm_bandwidth[0:-1])
except ValueError: except ValueError:
CONLOG.error('Invalid --bandwidth parameter. A valid input must ' LOG.error('Invalid --bandwidth parameter. A valid input must '
'specify only one unit (K|M|G).') 'specify only one unit (K|M|G).')
sys.exit(1) sys.exit(1)
config.vm_bandwidth = int(val * (10 ** (ex_unit * 3))) config.vm_bandwidth = int(val * (10 ** (ex_unit * 3)))
@ -950,8 +959,8 @@ def merge_opts_to_configs(opts):
for i in xrange(len(config.tcp_pkt_sizes)): for i in xrange(len(config.tcp_pkt_sizes)):
config.tcp_pkt_sizes[i] = int(config.tcp_pkt_sizes[i]) config.tcp_pkt_sizes[i] = int(config.tcp_pkt_sizes[i])
except ValueError: except ValueError:
CONLOG.error('Invalid --tcpbuf parameter. A valid input must be ' LOG.error('Invalid --tcpbuf parameter. A valid input must be '
'integers seperated by comma.') 'integers seperated by comma.')
sys.exit(1) sys.exit(1)
if opts.udp_pkt_sizes: if opts.udp_pkt_sizes:
@ -960,8 +969,8 @@ def merge_opts_to_configs(opts):
for i in xrange(len(config.udp_pkt_sizes)): for i in xrange(len(config.udp_pkt_sizes)):
config.udp_pkt_sizes[i] = int(config.udp_pkt_sizes[i]) config.udp_pkt_sizes[i] = int(config.udp_pkt_sizes[i])
except ValueError: except ValueError:
CONLOG.error('Invalid --udpbuf parameter. A valid input must be ' LOG.error('Invalid --udpbuf parameter. A valid input must be '
'integers seperated by comma.') 'integers seperated by comma.')
sys.exit(1) sys.exit(1)
if opts.reuse_network_name: if opts.reuse_network_name:
@ -987,12 +996,12 @@ def merge_opts_to_configs(opts):
if mobj: if mobj:
config.gmond_svr_ip = mobj.group(1) config.gmond_svr_ip = mobj.group(1)
config.gmond_svr_port = mobj.group(2) config.gmond_svr_port = mobj.group(2)
CONLOG.info("Ganglia monitoring enabled (%s:%s)" % LOG.info("Ganglia monitoring enabled (%s:%s)" %
(config.gmond_svr_ip, config.gmond_svr_port)) (config.gmond_svr_ip, config.gmond_svr_port))
config.time = 30 config.time = 30
else: else:
CONLOG.warning('Invalid --monitor syntax: ' + opts.monitor) LOG.warning('Invalid --monitor syntax: ' + opts.monitor)
################################################### ###################################################
# Once we parse the config files, normalize # Once we parse the config files, normalize
@ -1017,7 +1026,7 @@ def merge_opts_to_configs(opts):
elif opts.tp_tool.lower() == 'iperf': elif opts.tp_tool.lower() == 'iperf':
config.tp_tool = iperf_tool.IperfTool config.tp_tool = iperf_tool.IperfTool
else: else:
CONLOG.warning('Invalid transport tool: ' + opts.tp_tool) LOG.warning('Invalid transport tool: ' + opts.tp_tool)
sys.exit(1) sys.exit(1)
else: else:
config.tp_tool = None config.tp_tool = None
@ -1068,12 +1077,13 @@ def run_vmtp(opts):
for item in native_tp_results: for item in native_tp_results:
rescol.add_flow_result(item) rescol.add_flow_result(item)
CONLOG.info(rescol.ppr.pformat(item)) CONLOG.info(rescol.ppr.pformat(item))
FILELOG.info(json.dumps(item, sort_keys=True))
# Parse the credentials of the OpenStack cloud, and run the benchmarking # Parse the credentials of the OpenStack cloud, and run the benchmarking
cred = credentials.Credentials(opts.rc, opts.passwd, opts.no_env) cred = credentials.Credentials(opts.rc, opts.passwd, opts.no_env)
if cred.rc_auth_url: if cred.rc_auth_url:
if config.debug: if config.debug:
CONLOG.info('Using ' + cred.rc_auth_url) LOG.info('Using ' + cred.rc_auth_url)
vmtp_instance = VmtpTest(config, cred, rescol) vmtp_instance = VmtpTest(config, cred, rescol)
vmtp_instance.run() vmtp_instance.run()
vmtp_net = vmtp_instance.net vmtp_net = vmtp_instance.net