Feature commit to support storage testing on ephemeral disk

1. Feature commit to support storage testing on ephemeral disk;
2. Default the disk size to 0;
3. Update to fio-2.8 in KloudBuster image;

Change-Id: I72b61828f3ce04c48ab9ab107e37a4927c3a454f

kb_dib/elements/kloudbuster/post-install.d/01-kb-script

@@ -79,7 +79,7 @@ mv /usr/local/lib/libhdr_histogram.so /usr/lib/
 
 # Install fio
 cd /tmp
-FIO='fio-2.6'
+FIO='fio-2.8'
 wget http://brick.kernel.dk/snaps/$FIO.tar.gz
 tar xzf $FIO.tar.gz
 cd $FIO

kb_dib/elements/kloudbuster/static/kb_test/kb_vm_agent.py

@@ -116,12 +116,12 @@ class KB_Instance(object):
 
     # Init volume
     @staticmethod
-    def init_volume(dest_path, size):
+    def init_volume(dest_path, size, mkfs):
-        cmd = 'if [ ! -e /mnt/volume ]; then\n'
+        cmd = 'if [ ! -e /kb_mnt ]; then\n'
-        cmd += 'mkfs.xfs /dev/vdb && '
+        cmd += 'mkfs.xfs /dev/vdb && ' if mkfs else ''
-        cmd += 'mkdir -p /mnt/volume && '
+        cmd += 'mkdir -p /kb_mnt && '
-        cmd += 'mount /dev/vdb /mnt/volume && '
+        cmd += 'mount /dev/vdb /kb_mnt && '
-        cmd += '%s --name=create_file --filename=/mnt/volume/kb_storage_test.bin '\
+        cmd += '%s --name=create_file --filename=/kb_mnt/kb_storage_test.bin '\
                '--size=%s --create_only=1\n' % (dest_path, size)
         cmd += 'fi'
         return cmd
@@ -131,7 +131,7 @@ class KB_Instance(object):
     def run_fio(dest_path, name, description, mode, block_size, iodepth, runtime,
                 rate_iops=None, rate=None, rwmixread=None, status_interval=None, extra_opts=None):
         fixed_opt = '--thread --ioengine=libaio --output-format=json+ --direct=1 '
-        fixed_opt += '--filename=/mnt/volume/kb_storage_test.bin '
+        fixed_opt += '--filename=/kb_mnt/kb_storage_test.bin '
         required_opt = '--name=%s --rw=%s --bs=%s --iodepth=%s --runtime=%s ' %\
             (name, mode, block_size, iodepth, runtime)
         optional_opt = ''
@@ -328,8 +328,10 @@ class KBA_Storage_Client(KBA_Client):
 
         return json.dumps(p_output)
 
-    def exec_init_volume(self, size):
+    def exec_init_volume(self, vol_init_configs):
-        self.last_cmd = KB_Instance.init_volume('/usr/local/bin/fio', size)
+        self.last_cmd = KB_Instance.init_volume(
+            dest_path='/usr/local/bin/fio',
+            **vol_init_configs)
         return self.exec_command(self.last_cmd)
 
     def exec_run_storage_test(self, fio_configs):

kloudbuster/base_compute.py

@@ -246,11 +246,12 @@ class Flavor(object):
     def list(self):
         return self.novaclient.flavors.list()
 
-    def create_flavor(self, name, ram, vcpus, disk, override=False):
+    def create_flavor(self, name, ram, vcpus, disk, ephemeral, override=False):
         # Creating flavors
         if override:
             self.delete_flavor(name)
-        return self.novaclient.flavors.create(name=name, ram=ram, vcpus=vcpus, disk=disk)
+        return self.novaclient.flavors.create(name=name, ram=ram, vcpus=vcpus,
+                                              disk=disk, ephemeral=ephemeral)
 
     def delete_flavor(self, name):
         try:

kloudbuster/base_network.py

@@ -126,9 +126,11 @@ class BaseNetwork(object):
         vm_total = config_scale['vms_per_network']
         if config_scale['use_floatingip']:
             external_network = find_external_network(self.neutron_client)
-        if config_scale.get('volume_size'):
+        if config_scale.get('storage_target') == 'volume' and config_scale.get('disk_size'):
             bs_obj = base_storage.BaseStorage(self.cinder_client)
-            vol_size = config_scale['volume_size']
+            vol_size = config_scale['disk_size']
+        else:
+            vol_size = 0
         # Schedule to create the required number of VMs
         for instance_count in xrange(vm_total):
             vm_name = network_prefix + "-I" + str(instance_count)
@@ -137,7 +139,7 @@ class BaseNetwork(object):
 
             # Create volume if needed
             # Don't create volumn for KB-Proxy
-            if config_scale.get('volume_size') and instance_count < vm_total - 1:
+            if vol_size and instance_count < vm_total - 1:
                 vol_name = network_prefix + "-V" + str(instance_count)
                 perf_instance.vol = bs_obj.create_vol(vol_size, name=vol_name)
                 self.res_logger.log('volumes', vol_name, perf_instance.vol.id)

kloudbuster/cfg.scale.yaml

@@ -54,7 +54,7 @@ server:
         # Memory for the flavor in MB
         ram: 2048
         # Size of local disk in GB
-        disk: 20
+        disk: 0
 
     # Number of tenants to be created on the cloud
     # KloudBuster will also create 1 user automatically for each tenant
@@ -165,7 +165,7 @@ client:
         # Memory for the flavor in MB
         ram: 2048
         # Size of local disk in GB
-        disk: 20
+        disk: 0
 
     # Traffic shaping - VM Placement hint
     # Availability zone to use for clients in the client cloud
@@ -280,11 +280,18 @@ client:
           rate: '60M'
           rwmixread: 70
 
-    # Volumes size in GB for each VM
+    # KloudBuster supports to run storage tests on Cinder Volumes or Ephemeral
-    # Will effect only in storage testing mode
+    # Disks. Available options to be configured: ['volume', 'ephemeral'].
-    volume_size: 1
+    storage_target: 'volume'
 
-    # The size of the test file for doing IO tests in GiB
+    # Volumes size in GB for each VM
+    #
+    # Note: Will effect only in storage testing.
+    disk_size: 1
+
+    # The size of the test file for running IO tests in GiB. Must be less or
+    # equal than disk_size.
+    #
     # Note: Due to the unit calculation difference and the overhead of the
     # filesystem, the actual available space for data will be smaller than the
     # volume size. So we picked GiB as the unit for laying out the test file,

kloudbuster/kb_config.py

@@ -97,14 +97,17 @@ class KBConfig(object):
                 LOG.warning('No public key is found or specified to instantiate VMs. '
                             'You will not be able to access the VMs spawned by KloudBuster.')
 
-        if self.storage_mode and not self.config_scale.client['volume_size']:
+        if self.storage_mode:
-            LOG.error('You have to specify a volumn size in order to run '
+            if not self.config_scale.client.disk_size:
-                      'storage performance tests.')
+                LOG.error('You have to specify a disk size in order to run storage tests.')
                 raise KBConfigParseException()
 
-        if not self.storage_mode:
+            if self.config_scale.client.io_file_size > self.config_scale.client.disk_size:
+                LOG.error('io_file_size must be less or eqaul than disk_size.')
+                raise KBConfigParseException()
+        else:
             # Ignore volume_size if not performing storage testing
-            self.config_scale['client']['volume_size'] = 0
+            self.config_scale['client']['disk_size'] = 0
 
         if self.alt_cfg:
             self.config_scale = self.config_scale + AttrDict(self.alt_cfg)

kloudbuster/kb_runner_storage.py

@@ -71,8 +71,11 @@ class KBRunner_Storage(KBRunner):
         return msg
 
     def init_volume(self, active_range, timeout=30):
+        parameter = {'size': str(self.config.io_file_size) + 'GiB'}
+        parameter['mkfs'] = True if self.config.storage_target == 'volume' else False
+
         func = {'cmd': 'init_volume', 'active_range': active_range,
-                'parameter': str(self.config.io_file_size) + 'GiB'}
+                'parameter': parameter}
         self.send_cmd('EXEC', 'storage', func)
         cnt_succ = self.polling_vms(timeout)[0]
         if cnt_succ != len(self.client_dict):
@@ -96,7 +99,10 @@ class KBRunner_Storage(KBRunner):
     def single_run(self, active_range=None, test_only=False):
         try:
             if not test_only:
-                LOG.info("Initilizing volume and setting up filesystem...")
+                if self.config.storage_target == 'volume':
+                    LOG.info("Initializing volume and setting up filesystem...")
+                else:
+                    LOG.info("Initializing ephermeral disk...")
                 self.init_volume(active_range)
 
                 if self.config.prompt_before_run:

kloudbuster/kloudbuster.py

@@ -109,11 +109,13 @@ class Kloud(object):
             nova_client = self.tenant_list[0].user_list[0].nova_client
             flavor_manager = base_compute.Flavor(nova_client)
             flavor_dict = self.scale_cfg.flavor
+            if self.scale_cfg.get('storage_target') == 'ephemeral':
+                flavor_dict['ephemeral'] = self.scale_cfg.get('disk_size')
             if self.testing_side:
                 flv = flavor_manager.create_flavor('KB.client', override=True, **flavor_dict)
                 self.res_logger.log('flavors', vars(flv)['name'], vars(flv)['id'])
                 flv = flavor_manager.create_flavor('KB.proxy', override=True,
-                                                   ram=2048, vcpus=1, disk=20)
+                                                   ram=2048, vcpus=1, disk=0, ephemeral=0)
                 self.res_logger.log('flavors', vars(flv)['name'], vars(flv)['id'])
             else:
                 flv = flavor_manager.create_flavor('KB.server', override=True, **flavor_dict)