x/kloudbuster
Code Issues Proposed changes

Feature commit to support storage testing on ephemeral disk

Browse Source 
1. Feature commit to support storage testing on ephemeral disk;
2. Default the disk size to 0;
3. Update to fio-2.8 in KloudBuster image;

Change-Id: I72b61828f3ce04c48ab9ab107e37a4927c3a454f
This commit is contained in:
Yichen Wang 2016-03-21 17:30:17 -07:00
parent c0dd63c370
commit 00ca0da14e
8 changed files with 53 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
kb_dib/elements/kloudbuster/post-install.d/01-kb-script
View File

@ -79,7 +79,7 @@ mv /usr/local/lib/libhdr_histogram.so /usr/lib/
# Install fio
cd /tmp
FIO='fio-2.6'
FIO='fio-2.8'
wget http://brick.kernel.dk/snaps/$FIO.tar.gz
tar xzf $FIO.tar.gz
cd $FIO

20
kb_dib/elements/kloudbuster/static/kb_test/kb_vm_agent.py
View File

@ -116,12 +116,12 @@ class KB_Instance(object):
# Init volume
@staticmethod
def init_volume(dest_path, size):
cmd = 'if [ ! -e /mnt/volume ]; then\n'
cmd += 'mkfs.xfs /dev/vdb && '
cmd += 'mkdir -p /mnt/volume && '
cmd += 'mount /dev/vdb /mnt/volume && '
cmd += '%s --name=create_file --filename=/mnt/volume/kb_storage_test.bin '\
def init_volume(dest_path, size, mkfs):
cmd = 'if [ ! -e /kb_mnt ]; then\n'
cmd += 'mkfs.xfs /dev/vdb && ' if mkfs else ''
cmd += 'mkdir -p /kb_mnt && '
cmd += 'mount /dev/vdb /kb_mnt && '
cmd += '%s --name=create_file --filename=/kb_mnt/kb_storage_test.bin '\
'--size=%s --create_only=1\n' % (dest_path, size)
cmd += 'fi'
return cmd
@ -131,7 +131,7 @@ class KB_Instance(object):
def run_fio(dest_path, name, description, mode, block_size, iodepth, runtime,
rate_iops=None, rate=None, rwmixread=None, status_interval=None, extra_opts=None):
fixed_opt = '--thread --ioengine=libaio --output-format=json+ --direct=1 '
fixed_opt += '--filename=/mnt/volume/kb_storage_test.bin '
fixed_opt += '--filename=/kb_mnt/kb_storage_test.bin '
required_opt = '--name=%s --rw=%s --bs=%s --iodepth=%s --runtime=%s ' %\
(name, mode, block_size, iodepth, runtime)
optional_opt = ''
@ -328,8 +328,10 @@ class KBA_Storage_Client(KBA_Client):
return json.dumps(p_output)
def exec_init_volume(self, size):
self.last_cmd = KB_Instance.init_volume('/usr/local/bin/fio', size)
def exec_init_volume(self, vol_init_configs):
self.last_cmd = KB_Instance.init_volume(
dest_path='/usr/local/bin/fio',
**vol_init_configs)
return self.exec_command(self.last_cmd)
def exec_run_storage_test(self, fio_configs):

5
kloudbuster/base_compute.py
View File

@ -246,11 +246,12 @@ class Flavor(object):
def list(self):
return self.novaclient.flavors.list()
def create_flavor(self, name, ram, vcpus, disk, override=False):
def create_flavor(self, name, ram, vcpus, disk, ephemeral, override=False):
# Creating flavors
if override:
self.delete_flavor(name)
return self.novaclient.flavors.create(name=name, ram=ram, vcpus=vcpus, disk=disk)
return self.novaclient.flavors.create(name=name, ram=ram, vcpus=vcpus,
disk=disk, ephemeral=ephemeral)
def delete_flavor(self, name):
try:

8
kloudbuster/base_network.py
View File

@ -126,9 +126,11 @@ class BaseNetwork(object):
vm_total = config_scale['vms_per_network']
if config_scale['use_floatingip']:
external_network = find_external_network(self.neutron_client)
if config_scale.get('volume_size'):
if config_scale.get('storage_target') == 'volume' and config_scale.get('disk_size'):
bs_obj = base_storage.BaseStorage(self.cinder_client)
vol_size = config_scale['volume_size']
vol_size = config_scale['disk_size']
else:
vol_size = 0
# Schedule to create the required number of VMs
for instance_count in xrange(vm_total):
vm_name = network_prefix + "-I" + str(instance_count)
@ -137,7 +139,7 @@ class BaseNetwork(object):
# Create volume if needed
# Don't create volumn for KB-Proxy
if config_scale.get('volume_size') and instance_count < vm_total - 1:
if vol_size and instance_count < vm_total - 1:
vol_name = network_prefix + "-V" + str(instance_count)
perf_instance.vol = bs_obj.create_vol(vol_size, name=vol_name)
self.res_logger.log('volumes', vol_name, perf_instance.vol.id)

19
kloudbuster/cfg.scale.yaml
View File

@ -54,7 +54,7 @@ server:
# Memory for the flavor in MB
ram: 2048
# Size of local disk in GB
disk: 20
disk: 0
# Number of tenants to be created on the cloud
# KloudBuster will also create 1 user automatically for each tenant
@ -165,7 +165,7 @@ client:
# Memory for the flavor in MB
ram: 2048
# Size of local disk in GB
disk: 20
disk: 0
# Traffic shaping - VM Placement hint
# Availability zone to use for clients in the client cloud
@ -280,11 +280,18 @@ client:
rate: '60M'
rwmixread: 70
# Volumes size in GB for each VM
# Will effect only in storage testing mode
volume_size: 1
# KloudBuster supports to run storage tests on Cinder Volumes or Ephemeral
# Disks. Available options to be configured: ['volume', 'ephemeral'].
storage_target: 'volume'
# The size of the test file for doing IO tests in GiB
# Volumes size in GB for each VM
#
# Note: Will effect only in storage testing.
disk_size: 1
# The size of the test file for running IO tests in GiB. Must be less or
# equal than disk_size.
#
# Note: Due to the unit calculation difference and the overhead of the
# filesystem, the actual available space for data will be smaller than the
# volume size. So we picked GiB as the unit for laying out the test file,

13
kloudbuster/kb_config.py
View File

@ -97,14 +97,17 @@ class KBConfig(object):
LOG.warning('No public key is found or specified to instantiate VMs. '
'You will not be able to access the VMs spawned by KloudBuster.')
if self.storage_mode and not self.config_scale.client['volume_size']:
LOG.error('You have to specify a volumn size in order to run '
'storage performance tests.')
if self.storage_mode:
if not self.config_scale.client.disk_size:
LOG.error('You have to specify a disk size in order to run storage tests.')
raise KBConfigParseException()
if not self.storage_mode:
if self.config_scale.client.io_file_size > self.config_scale.client.disk_size:
LOG.error('io_file_size must be less or eqaul than disk_size.')
raise KBConfigParseException()
else:
# Ignore volume_size if not performing storage testing
self.config_scale['client']['volume_size'] = 0
self.config_scale['client']['disk_size'] = 0
if self.alt_cfg:
self.config_scale = self.config_scale + AttrDict(self.alt_cfg)

10
kloudbuster/kb_runner_storage.py
View File

@ -71,8 +71,11 @@ class KBRunner_Storage(KBRunner):
return msg
def init_volume(self, active_range, timeout=30):
parameter = {'size': str(self.config.io_file_size) + 'GiB'}
parameter['mkfs'] = True if self.config.storage_target == 'volume' else False
func = {'cmd': 'init_volume', 'active_range': active_range,
'parameter': str(self.config.io_file_size) + 'GiB'}
'parameter': parameter}
self.send_cmd('EXEC', 'storage', func)
cnt_succ = self.polling_vms(timeout)[0]
if cnt_succ != len(self.client_dict):
@ -96,7 +99,10 @@ class KBRunner_Storage(KBRunner):
def single_run(self, active_range=None, test_only=False):
try:
if not test_only:
LOG.info("Initilizing volume and setting up filesystem...")
if self.config.storage_target == 'volume':
LOG.info("Initializing volume and setting up filesystem...")
else:
LOG.info("Initializing ephermeral disk...")
self.init_volume(active_range)
if self.config.prompt_before_run:

4
kloudbuster/kloudbuster.py
View File

@ -109,11 +109,13 @@ class Kloud(object):
nova_client = self.tenant_list[0].user_list[0].nova_client
flavor_manager = base_compute.Flavor(nova_client)
flavor_dict = self.scale_cfg.flavor
if self.scale_cfg.get('storage_target') == 'ephemeral':
flavor_dict['ephemeral'] = self.scale_cfg.get('disk_size')
if self.testing_side:
flv = flavor_manager.create_flavor('KB.client', override=True, **flavor_dict)
self.res_logger.log('flavors', vars(flv)['name'], vars(flv)['id'])
flv = flavor_manager.create_flavor('KB.proxy', override=True,
ram=2048, vcpus=1, disk=20)
ram=2048, vcpus=1, disk=0, ephemeral=0)
self.res_logger.log('flavors', vars(flv)['name'], vars(flv)['id'])
else:
flv = flavor_manager.create_flavor('KB.server', override=True, **flavor_dict)