diff --git a/.gitignore b/.gitignore index 2f13908..1707d7e 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,5 @@ kb*.log *.html *.qcow2 scale/dib/kloudbuster.d/ + +.vagrant/ diff --git a/kloudbuster/cfg.scale.yaml b/kloudbuster/cfg.scale.yaml index ea2a3d8..490e4a9 100644 --- a/kloudbuster/cfg.scale.yaml +++ b/kloudbuster/cfg.scale.yaml @@ -2,8 +2,12 @@ # Name of the image to use for all test VMs (client, server and proxy) # The image name must exist in OpenStack and must be built with the appropriate -# packages -image_name: 'KloudBuster Image' +# packages. +# The default test VM image is named "kloudbuster_" where +# is the KloudBuster test VM image version (e.g. "kloudbuster_v3") +# Leave empty to use the default test VM image (recommended). +# If non empty use quotes if there are space characters in the name (e.g. 'my image') +image_name: # Config options common to client and server side keystone_admin_role: "admin" diff --git a/kloudbuster/dib/Vagrantfile b/kloudbuster/dib/Vagrantfile index 63ab902..251f8e3 100644 --- a/kloudbuster/dib/Vagrantfile +++ b/kloudbuster/dib/Vagrantfile @@ -15,15 +15,25 @@ apt-get -y install qemu-utils git clone git://github.com/openstack/diskimage-builder.git git clone git://github.com/openstack/dib-utils.git # install kloudbuster -git clone -b kloudbuster git://github.com/stackforge/vmtp.git +git clone git://github.com/openstack/kloudbuster.git + +kb_root=kloudbuster/kloudbuster +# Extract image version number '__version__ = 2.0' becomes '__version__=2_0' +ver=`grep '^__version__' $kb_root/kb_vm_agent.py | tr -d ' ' | tr '.' '_'` +eval $ver + +kb_image_name=kloudbuster_v$__version__ + +echo "Building $kb_image_name.qcow2..." # Add diskimage-builder and dib-utils bin to the path export PATH=$PATH:`pwd`/diskimage-builder/bin:`pwd`/dib-utils/bin # Add the kloudbuster elements directory to the DIB elements path -export ELEMENTS_PATH=`pwd`/vmtp/scale/dib/elements -time disk-image-create -o kloudbuster ubuntu kloudbuster -mv kloudbuster.qcow2 /vagrant +export ELEMENTS_PATH=`pwd`/$kb_root/dib/elements +time disk-image-create -o $kb_image_name ubuntu kloudbuster +mv $kb_image_name.qcow2 /vagrant + SCRIPT # All Vagrant configuration is done below. The "2" in Vagrant.configure @@ -75,16 +85,7 @@ Vagrant.configure(2) do |config| # Customize the amount of memory on the VM: vb.memory = "2048" end - # - # View the documentation for the provider you are using for more - # information on available options. - # Define a Vagrant Push strategy for pushing to Atlas. Other push strategies - # such as FTP and Heroku are also available. See the documentation at - # https://docs.vagrantup.com/v2/push/atlas.html for more information. - # config.push.define "atlas" do |push| - # push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME" - # end # Enable provisioning with a shell script. Additional provisioners such as # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the diff --git a/kloudbuster/dib/build-image.sh b/kloudbuster/dib/build-image.sh index 8a6484f..08f10cc 100755 --- a/kloudbuster/dib/build-image.sh +++ b/kloudbuster/dib/build-image.sh @@ -17,9 +17,17 @@ export PATH=$PATH:`pwd`/diskimage-builder/bin:`pwd`/dib-utils/bin # Add the kloudbuster elements directory to the DIB elements path export ELEMENTS_PATH=`pwd`/elements -time disk-image-create -o kloudbuster ubuntu kloudbuster +# Extract image version number '__version__ = 2.0' becomes '__version__=2_0' +ver=`grep '^__version__' ../kb_vm_agent.py | tr -d ' ' | tr '.' '_'` +eval $ver -ls -l kloudbuster.qcow2 +kb_image_name=kloudbuster_v$__version__ + +echo "Building $kb_image_name.qcow2..." + +time disk-image-create -o $kb_image_name ubuntu kloudbuster + +ls -l $kb_image_name.qcow2 # cleanup rm -rf diskimage-builder dib-utils diff --git a/kloudbuster/dib/elements/kloudbuster/static/kb_test/kb_vm_agent.py b/kloudbuster/dib/elements/kloudbuster/static/kb_test/kb_vm_agent.py index 7b53a14..4982c6f 100644 --- a/kloudbuster/dib/elements/kloudbuster/static/kb_test/kb_vm_agent.py +++ b/kloudbuster/dib/elements/kloudbuster/static/kb_test/kb_vm_agent.py @@ -20,15 +20,27 @@ import time import redis -# Define the version of the KloudBuster agent. +# Define the version of the KloudBuster agent and VM image # # When VM is up running, the agent will send the READY message to the # KloudBuster main program, along with its version. The main program # will check the version to see whether the image meets the minimum # requirements to run, and stopped with an error if not. # -# Note: All majors are compatible regardless of minor. -__version__ = '2.0' +# This version must be incremented if the interface changes or if new features +# are added to the agent VM +__version__ = '3' + +def get_image_name(): + '''Return the versioned VM image name that corresponds to this + agent code. This string must match the way DIB names the kloudbuster image. + Return: + the versioned image name without the extension ('.qcow2' is implicit) + ''' + return 'kloudbuster_v' + __version__ + +def get_image_version(): + return __version__ class KB_Instance(object): diff --git a/kloudbuster/kb_config.py b/kloudbuster/kb_config.py index c482301..19073fc 100644 --- a/kloudbuster/kb_config.py +++ b/kloudbuster/kb_config.py @@ -143,8 +143,12 @@ class KBConfig(object): def get_configs(self): if CONF.config: - alt_config = configure.Configuration.from_file(CONF.config).configure() - self.config_scale = self.config_scale.merge(alt_config) + try: + alt_config = configure.Configuration.from_file(CONF.config).configure() + self.config_scale = self.config_scale.merge(alt_config) + except configure.ConfigurationError: + # file can be empty + pass def get_topo_cfg(self): if CONF.topology: diff --git a/kloudbuster/kb_runner.py b/kloudbuster/kb_runner.py index 41144be..c3a6471 100644 --- a/kloudbuster/kb_runner.py +++ b/kloudbuster/kb_runner.py @@ -14,12 +14,16 @@ from collections import deque from distutils.version import LooseVersion +from sets import Set import threading import time import log as logging import redis +# A set of warned VM version mismatches +vm_version_mismatches = Set() + LOG = logging.getLogger(__name__) class KBVMUpException(Exception): @@ -42,14 +46,14 @@ class KBRunner(object): Control the testing VMs on the testing cloud """ - def __init__(self, client_list, config, required_agent_version, single_cloud=True): + def __init__(self, client_list, config, expected_agent_version, single_cloud=True): self.client_dict = dict(zip([x.vm_name for x in client_list], client_list)) self.config = config self.single_cloud = single_cloud self.result = {} self.host_stats = {} self.tool_result = {} - self.required_agent_version = str(required_agent_version) + self.expected_agent_version = expected_agent_version self.agent_version = None # Redis @@ -232,12 +236,14 @@ class KBRunner(object): LOG.info("Waiting for agents on VMs to come up...") self.wait_for_vm_up() if not self.agent_version: - self.agent_version = "0.0" - if (LooseVersion(self.agent_version) < LooseVersion(self.required_agent_version)): - LOG.error("The VM image you are running is too old (%s), the minimum version " - "required is %s.x. Please build the image from latest repository." % - (self.agent_version, self.required_agent_version)) - return + self.agent_version = "0" + if (LooseVersion(self.agent_version) != LooseVersion(self.expected_agent_version)): + # only warn once for each unexpected VM version + if self.expected_agent_version not in vm_version_mismatches: + vm_version_mismatches.add(self.expected_agent_version) + LOG.warn("The VM image you are running (%s) is not the expected version (%s) " + "this may cause some incompatibilities" % + (self.agent_version, self.expected_agent_version)) if self.single_cloud: LOG.info("Setting up static route to reach tested cloud...") self.setup_static_route() diff --git a/kloudbuster/kloudbuster.py b/kloudbuster/kloudbuster.py index 2acc657..78cdb68 100755 --- a/kloudbuster/kloudbuster.py +++ b/kloudbuster/kloudbuster.py @@ -29,6 +29,7 @@ from kb_config import KBConfig from kb_res_logger import KBResLogger from kb_runner import KBRunner from kb_scheduler import KBScheduler +import kb_vm_agent from keystoneclient.v2_0 import client as keystoneclient import log as logging from novaclient.exceptions import ClientException @@ -38,7 +39,6 @@ import tenant CONF = cfg.CONF LOG = logging.getLogger(__name__) -KB_IMAGE_MAJOR_VERSION = 2 class KBVMCreationException(Exception): pass @@ -237,6 +237,7 @@ class KloudBuster(object): keystone_list = [create_keystone_client(self.server_cred)[0], create_keystone_client(self.client_cred)[0]] keystone_dict = dict(zip(['Server kloud', 'Client kloud'], keystone_list)) + img_name_dict = dict(zip(['Server kloud', 'Client kloud'], [self.server_cfg.image_name, self.client_cfg.image_name])) @@ -252,12 +253,14 @@ class KloudBuster(object): pass # Trying to upload images - LOG.info("Image is not found in %s, trying to upload..." % (kloud)) - if not os.path.exists('dib/kloudbuster.qcow2'): - LOG.error("Image file dib/kloudbuster.qcow2 is not present, please refer " + kb_image_name = 'dib/' + kb_vm_agent.get_image_name() + '.qcow2' + if not os.path.exists(kb_image_name): + LOG.error("VM Image not in Glance and could not find " + kb_image_name + + " to upload, please refer " "to dib/README.rst for how to build image for KloudBuster.") return False - with open('dib/kloudbuster.qcow2') as fimage: + LOG.info("Image is not found in %s, uploading %s..." % (kloud, kb_image_name)) + with open(kb_image_name) as fimage: try: image = glance_client.images.create(name=img_name_dict[kloud], disk_format="qcow2", @@ -266,8 +269,9 @@ class KloudBuster(object): glance_client.images.upload(image['id'], fimage) except glance_exception.HTTPForbidden: LOG.error("Cannot upload image without admin access. Please make sure the " - "image is existed in cloud, and is either public or owned by you.") + "image is uploaded and is either public or owned by you.") return False + return True def print_provision_info(self): """ @@ -352,7 +356,7 @@ class KloudBuster(object): self.testing_kloud.create_vm(self.kb_proxy) kbrunner = KBRunner(client_list, self.client_cfg, - KB_IMAGE_MAJOR_VERSION, + kb_vm_agent.get_image_version(), self.single_cloud) kbrunner.setup_redis(self.kb_proxy.fip_ip) @@ -548,12 +552,23 @@ def main(): ] CONF.register_cli_opts(cli_opts) CONF.set_default("verbose", True) - CONF(sys.argv[1:], project="kloudbuster", version=__version__) + full_version = __version__ + ', VM image: ' + kb_vm_agent.get_image_name() + CONF(sys.argv[1:], project="kloudbuster", version=full_version) logging.setup("kloudbuster") + try: + kb_config = KBConfig() + kb_config.init_with_cli() + except TypeError: + LOG.error('Error parsing the configuration file') + sys.exit(1) - kb_config = KBConfig() - kb_config.init_with_cli() + # Use the default image name for Glance + # defaults to something like "kloudbuster_v3" + if not kb_config.server_cfg['image_name']: + kb_config.server_cfg['image_name'] = kb_vm_agent.get_image_name() + if not kb_config.client_cfg['image_name']: + kb_config.client_cfg['image_name'] = kb_vm_agent.get_image_name() # The KloudBuster class is just a wrapper class # levarages tenant and user class for resource creations and deletion diff --git a/requirements.txt b/requirements.txt index 9121fa4..5da73d9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ pbr>=1.3 Babel>=1.3 configure>=0.5 -hdrhistogram>=0.1.1 +hdrhistogram>=0.2.2 oslo.log>=1.0.0 pecan>=0.9.0 python-openstackclient>=1.5.0