Modify VM versioning and DIB image name

Relax runtime VM version check
Update hdrhistogram requirement

Change-Id: I68ad2a75c6256461b68b5f5d8e70055f4a22df16

.gitignore

@@ -60,3 +60,5 @@ kb*.log
 *.html
 *.qcow2
 scale/dib/kloudbuster.d/
+
+.vagrant/

kloudbuster/cfg.scale.yaml

@@ -2,8 +2,12 @@
 
 # Name of the image to use for all test VMs (client, server and proxy)
 # The image name must exist in OpenStack and must be built with the appropriate
-# packages
+# packages.
-image_name: 'KloudBuster Image'
+# The default test VM image is named "kloudbuster_<version>" where
+# <version> is the KloudBuster test VM image version (e.g. "kloudbuster_v3")
+# Leave empty to use the default test VM image (recommended).
+# If non empty use quotes if there are space characters in the name (e.g. 'my image')
+image_name:
 
 # Config options common to client and server side
 keystone_admin_role: "admin"

kloudbuster/dib/Vagrantfile

@@ -15,15 +15,25 @@ apt-get -y install qemu-utils
 git clone git://github.com/openstack/diskimage-builder.git
 git clone git://github.com/openstack/dib-utils.git
 # install kloudbuster
-git clone -b kloudbuster git://github.com/stackforge/vmtp.git
+git clone git://github.com/openstack/kloudbuster.git
+
+kb_root=kloudbuster/kloudbuster
+# Extract image version number '__version__ = 2.0' becomes '__version__=2_0'
+ver=`grep '^__version__' $kb_root/kb_vm_agent.py | tr -d ' ' | tr '.' '_'`
+eval $ver
+
+kb_image_name=kloudbuster_v$__version__
+
+echo "Building $kb_image_name.qcow2..."
 
 # Add diskimage-builder and dib-utils bin to the path
 export PATH=$PATH:`pwd`/diskimage-builder/bin:`pwd`/dib-utils/bin
 
 # Add the kloudbuster elements directory to the DIB elements path
-export ELEMENTS_PATH=`pwd`/vmtp/scale/dib/elements
+export ELEMENTS_PATH=`pwd`/$kb_root/dib/elements
-time disk-image-create -o kloudbuster ubuntu kloudbuster
+time disk-image-create -o $kb_image_name ubuntu kloudbuster
-mv kloudbuster.qcow2 /vagrant
+mv $kb_image_name.qcow2 /vagrant
+
 SCRIPT
 
 # All Vagrant configuration is done below. The "2" in Vagrant.configure
@@ -75,16 +85,7 @@ Vagrant.configure(2) do |config|
     # Customize the amount of memory on the VM:
     vb.memory = "2048"
   end
-  #
-  # View the documentation for the provider you are using for more
-  # information on available options.
 
-  # Define a Vagrant Push strategy for pushing to Atlas. Other push strategies
-  # such as FTP and Heroku are also available. See the documentation at
-  # https://docs.vagrantup.com/v2/push/atlas.html for more information.
-  # config.push.define "atlas" do |push|
-  #   push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME"
-  # end
 
   # Enable provisioning with a shell script. Additional provisioners such as
   # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the

kloudbuster/dib/build-image.sh

@@ -17,9 +17,17 @@ export PATH=$PATH:`pwd`/diskimage-builder/bin:`pwd`/dib-utils/bin
 # Add the kloudbuster elements directory to the DIB elements path
 export ELEMENTS_PATH=`pwd`/elements
 
-time disk-image-create -o kloudbuster ubuntu kloudbuster
+# Extract image version number '__version__ = 2.0' becomes '__version__=2_0'
+ver=`grep '^__version__' ../kb_vm_agent.py | tr -d ' ' | tr '.' '_'`
+eval $ver
 
-ls -l kloudbuster.qcow2
+kb_image_name=kloudbuster_v$__version__
+
+echo "Building $kb_image_name.qcow2..."
+
+time disk-image-create -o $kb_image_name ubuntu kloudbuster
+
+ls -l $kb_image_name.qcow2
 
 # cleanup
 rm -rf diskimage-builder dib-utils

kloudbuster/dib/elements/kloudbuster/static/kb_test/kb_vm_agent.py

@@ -20,15 +20,27 @@ import time
 
 import redis
 
-# Define the version of the KloudBuster agent.
+# Define the version of the KloudBuster agent and VM image
 #
 # When VM is up running, the agent will send the READY message to the
 # KloudBuster main program, along with its version. The main program
 # will check the version to see whether the image meets the minimum
 # requirements to run, and stopped with an error if not.
 #
-# Note: All majors are compatible regardless of minor.
+# This version must be incremented if the interface changes or if new features
-__version__ = '2.0'
+# are added to the agent VM
+__version__ = '3'
+
+def get_image_name():
+    '''Return the versioned VM image name that corresponds to this
+    agent code. This string must match the way DIB names the kloudbuster image.
+    Return:
+        the versioned image name without the extension ('.qcow2' is implicit)
+    '''
+    return 'kloudbuster_v' + __version__
+
+def get_image_version():
+    return __version__
 
 class KB_Instance(object):
 

kloudbuster/kb_config.py

@@ -143,8 +143,12 @@ class KBConfig(object):
 
     def get_configs(self):
         if CONF.config:
-            alt_config = configure.Configuration.from_file(CONF.config).configure()
+            try:
-            self.config_scale = self.config_scale.merge(alt_config)
+                alt_config = configure.Configuration.from_file(CONF.config).configure()
+                self.config_scale = self.config_scale.merge(alt_config)
+            except configure.ConfigurationError:
+                # file can be empty
+                pass
 
     def get_topo_cfg(self):
         if CONF.topology:

kloudbuster/kb_runner.py

@@ -14,12 +14,16 @@
 
 from collections import deque
 from distutils.version import LooseVersion
+from sets import Set
 import threading
 import time
 
 import log as logging
 import redis
 
+# A set of warned VM version mismatches
+vm_version_mismatches = Set()
+
 LOG = logging.getLogger(__name__)
 
 class KBVMUpException(Exception):
@@ -42,14 +46,14 @@ class KBRunner(object):
     Control the testing VMs on the testing cloud
     """
 
-    def __init__(self, client_list, config, required_agent_version, single_cloud=True):
+    def __init__(self, client_list, config, expected_agent_version, single_cloud=True):
         self.client_dict = dict(zip([x.vm_name for x in client_list], client_list))
         self.config = config
         self.single_cloud = single_cloud
         self.result = {}
         self.host_stats = {}
         self.tool_result = {}
-        self.required_agent_version = str(required_agent_version)
+        self.expected_agent_version = expected_agent_version
         self.agent_version = None
 
         # Redis
@@ -232,12 +236,14 @@ class KBRunner(object):
             LOG.info("Waiting for agents on VMs to come up...")
             self.wait_for_vm_up()
             if not self.agent_version:
-                self.agent_version = "0.0"
+                self.agent_version = "0"
-            if (LooseVersion(self.agent_version) < LooseVersion(self.required_agent_version)):
+            if (LooseVersion(self.agent_version) != LooseVersion(self.expected_agent_version)):
-                LOG.error("The VM image you are running is too old (%s), the minimum version "
+                # only warn once for each unexpected VM version
-                          "required is %s.x. Please build the image from latest repository." %
+                if self.expected_agent_version not in vm_version_mismatches:
-                          (self.agent_version, self.required_agent_version))
+                    vm_version_mismatches.add(self.expected_agent_version)
-                return
+                    LOG.warn("The VM image you are running (%s) is not the expected version (%s) "
+                             "this may cause some incompatibilities" %
+                             (self.agent_version, self.expected_agent_version))
             if self.single_cloud:
                 LOG.info("Setting up static route to reach tested cloud...")
                 self.setup_static_route()

kloudbuster/kloudbuster.py

@@ -29,6 +29,7 @@ from kb_config import KBConfig
 from kb_res_logger import KBResLogger
 from kb_runner import KBRunner
 from kb_scheduler import KBScheduler
+import kb_vm_agent
 from keystoneclient.v2_0 import client as keystoneclient
 import log as logging
 from novaclient.exceptions import ClientException
@@ -38,7 +39,6 @@ import tenant
 
 CONF = cfg.CONF
 LOG = logging.getLogger(__name__)
-KB_IMAGE_MAJOR_VERSION = 2
 
 class KBVMCreationException(Exception):
     pass
@@ -237,6 +237,7 @@ class KloudBuster(object):
         keystone_list = [create_keystone_client(self.server_cred)[0],
                          create_keystone_client(self.client_cred)[0]]
         keystone_dict = dict(zip(['Server kloud', 'Client kloud'], keystone_list))
+
         img_name_dict = dict(zip(['Server kloud', 'Client kloud'],
                                  [self.server_cfg.image_name, self.client_cfg.image_name]))
 
@@ -252,12 +253,14 @@ class KloudBuster(object):
                 pass
 
             # Trying to upload images
-            LOG.info("Image is not found in %s, trying to upload..." % (kloud))
+            kb_image_name = 'dib/' + kb_vm_agent.get_image_name() + '.qcow2'
-            if not os.path.exists('dib/kloudbuster.qcow2'):
+            if not os.path.exists(kb_image_name):
-                LOG.error("Image file dib/kloudbuster.qcow2 is not present, please refer "
+                LOG.error("VM Image not in Glance and could not find " + kb_image_name +
+                          " to upload, please refer "
                           "to dib/README.rst for how to build image for KloudBuster.")
                 return False
-            with open('dib/kloudbuster.qcow2') as fimage:
+            LOG.info("Image is not found in %s, uploading %s..." % (kloud, kb_image_name))
+            with open(kb_image_name) as fimage:
                 try:
                     image = glance_client.images.create(name=img_name_dict[kloud],
                                                         disk_format="qcow2",
@@ -266,8 +269,9 @@ class KloudBuster(object):
                     glance_client.images.upload(image['id'], fimage)
                 except glance_exception.HTTPForbidden:
                     LOG.error("Cannot upload image without admin access. Please make sure the "
-                              "image is existed in cloud, and is either public or owned by you.")
+                              "image is uploaded and is either public or owned by you.")
                     return False
+            return True
 
     def print_provision_info(self):
         """
@@ -352,7 +356,7 @@ class KloudBuster(object):
             self.testing_kloud.create_vm(self.kb_proxy)
 
             kbrunner = KBRunner(client_list, self.client_cfg,
-                                KB_IMAGE_MAJOR_VERSION,
+                                kb_vm_agent.get_image_version(),
                                 self.single_cloud)
             kbrunner.setup_redis(self.kb_proxy.fip_ip)
 
@@ -548,12 +552,23 @@ def main():
     ]
     CONF.register_cli_opts(cli_opts)
     CONF.set_default("verbose", True)
-    CONF(sys.argv[1:], project="kloudbuster", version=__version__)
+    full_version = __version__ + ', VM image: ' + kb_vm_agent.get_image_name()
+    CONF(sys.argv[1:], project="kloudbuster", version=full_version)
 
     logging.setup("kloudbuster")
+    try:
+        kb_config = KBConfig()
+        kb_config.init_with_cli()
+    except TypeError:
+        LOG.error('Error parsing the configuration file')
+        sys.exit(1)
 
-    kb_config = KBConfig()
+    # Use the default image name for Glance
-    kb_config.init_with_cli()
+    # defaults to something like "kloudbuster_v3"
+    if not kb_config.server_cfg['image_name']:
+        kb_config.server_cfg['image_name'] = kb_vm_agent.get_image_name()
+    if not kb_config.client_cfg['image_name']:
+        kb_config.client_cfg['image_name'] = kb_vm_agent.get_image_name()
 
     # The KloudBuster class is just a wrapper class
     # levarages tenant and user class for resource creations and deletion

requirements.txt

@@ -6,7 +6,7 @@ pbr>=1.3
 Babel>=1.3
 
 configure>=0.5
-hdrhistogram>=0.1.1
+hdrhistogram>=0.2.2
 oslo.log>=1.0.0
 pecan>=0.9.0
 python-openstackclient>=1.5.0