From b22f59b8c9666e00aa5cc4f8159c715754abbbf6 Mon Sep 17 00:00:00 2001 From: ahothan Date: Sat, 9 Sep 2017 08:57:03 -0700 Subject: [PATCH] Fix quota calculation for storage test [bug 1715333] https://bugs.launchpad.net/kloudbuster/+bug/1715333 Only calculate cinder quota for client tenant use the storage configs vm count and disk size (not the flavor disk size) Add INFO to display the calculated disk and volume quotas Only lookup for flavors ond image once at the kloud level (instead of every instance) Change-Id: Ic92d962d6765e682e3158d7b52d7c7f3310b09bc --- .testr.conf | 7 - LICENSE | 1 - MANIFEST.in | 1 - babel.cfg | 1 - kloudbuster/base_compute.py | 48 +++--- kloudbuster/kb_runner_base.py | 2 +- kloudbuster/kloudbuster.py | 300 +++++++++++++++++++++------------- kloudbuster/tenant.py | 28 ---- 8 files changed, 214 insertions(+), 174 deletions(-) delete mode 100644 .testr.conf diff --git a/.testr.conf b/.testr.conf deleted file mode 100644 index 6d83b3c..0000000 --- a/.testr.conf +++ /dev/null @@ -1,7 +0,0 @@ -[DEFAULT] -test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ - OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ - OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \ - ${PYTHON:-python} -m subunit.run discover -t ./ . $LISTOPT $IDOPTION -test_id_option=--load-list $IDFILE -test_list_option=--list diff --git a/LICENSE b/LICENSE index 68c771a..67db858 100644 --- a/LICENSE +++ b/LICENSE @@ -173,4 +173,3 @@ defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. - diff --git a/MANIFEST.in b/MANIFEST.in index c978a52..59c20d5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,5 +2,4 @@ include AUTHORS include ChangeLog exclude .gitignore exclude .gitreview - global-exclude *.pyc diff --git a/babel.cfg b/babel.cfg index 15cd6cb..efceab8 100644 --- a/babel.cfg +++ b/babel.cfg @@ -1,2 +1 @@ [python: **.py] - diff --git a/kloudbuster/base_compute.py b/kloudbuster/base_compute.py index 129fe58..ecff883 100644 --- a/kloudbuster/base_compute.py +++ b/kloudbuster/base_compute.py @@ -16,6 +16,7 @@ import os import time import log as logging +from novaclient.exceptions import BadRequest LOG = logging.getLogger(__name__) @@ -60,15 +61,14 @@ class BaseCompute(object): 5. Security group instance 6. Optional parameters: availability zone, user data, config drive """ - - # Get the image id and flavor id from their logical names - image = self.find_image(image_name) - flavor_type = self.find_flavor(flavor_type) + kloud = self.network.router.user.tenant.kloud + image = kloud.vm_img + flavor = kloud.flavors[flavor_type] # Also attach the created security group for the test instance = self.novaclient.servers.create(name=self.vm_name, image=image, - flavor=flavor_type, + flavor=flavor, key_name=keyname, nics=nic, availability_zone=avail_zone, @@ -118,17 +118,12 @@ class BaseCompute(object): if self.instance and self.vol: attached_vols = self.novaclient.volumes.get_server_volumes(self.instance.id) if len(attached_vols): - self.novaclient.volumes.delete_server_volume(self.instance.id, self.vol.id) - - def find_image(self, image_name): - """ - Given a image name return the image id - """ - try: - image = self.novaclient.glance.find_image(image_name) - return image - except Exception: - return None + try: + self.novaclient.volumes.delete_server_volume(self.instance.id, self.vol.id) + except BadRequest: + # WARNING Some resources in client cloud are not cleaned up properly.: + # BadRequest: Invalid volume: Volume must be attached in order to detach + pass def find_flavor(self, flavor_type): """ @@ -287,16 +282,21 @@ class Flavor(object): def list(self): return self.novaclient.flavors.list() - def create_flavor(self, name, ram, vcpus, disk, ephemeral, override=False): - # Creating flavors - if override: - self.delete_flavor(name) - return self.novaclient.flavors.create(name=name, ram=ram, vcpus=vcpus, - disk=disk, ephemeral=ephemeral) + def create_flavor(self, flavor_dict): + '''Delete the old flavor with same name if any and create a new one - def delete_flavor(self, name): + flavor_dict: dict with following keys: name, ram, vcpus, disk, ephemeral + ''' + name = flavor_dict['name'] + flavor = self.get(name) + if flavor: + LOG.info('Deleting old flavor %s', name) + self.delete_flavor(flavor) + LOG.info('Creating flavor %s', name) + return self.novaclient.flavors.create(**flavor_dict) + + def delete_flavor(self, flavor): try: - flavor = self.novaclient.flavors.find(name=name) flavor.delete() except Exception: pass diff --git a/kloudbuster/kb_runner_base.py b/kloudbuster/kb_runner_base.py index 97b99de..543ef08 100644 --- a/kloudbuster/kb_runner_base.py +++ b/kloudbuster/kb_runner_base.py @@ -183,7 +183,7 @@ class KBRunner(object): else: LOG.error('[%s] received invalid command: %s' + (vm_name, cmd)) - log_msg = "%d Succeed, %d Failed, %d Pending... Retry #%d" %\ + log_msg = "VMs: %d Ready, %d Failed, %d Pending... Retry #%d" %\ (cnt_succ, cnt_failed, len(clist), retry) if sample_count != 0: log_msg += " (%d sample(s) received)" % sample_count diff --git a/kloudbuster/kloudbuster.py b/kloudbuster/kloudbuster.py index 49849c4..d23e972 100755 --- a/kloudbuster/kloudbuster.py +++ b/kloudbuster/kloudbuster.py @@ -56,8 +56,17 @@ LOG = logging.getLogger(__name__) class KBVMCreationException(Exception): pass +class KBFlavorCheckException(Exception): + pass + +# flavor names to use +FLAVOR_KB_PROXY = 'KB.proxy' +FLAVOR_KB_CLIENT = 'KB.client' +FLAVOR_KB_SERVER = 'KB.server' + class Kloud(object): - def __init__(self, scale_cfg, cred, reusing_tenants, + + def __init__(self, scale_cfg, cred, reusing_tenants, vm_img, testing_side=False, storage_mode=False, multicast_mode=False): self.tenant_list = [] self.testing_side = testing_side @@ -67,9 +76,9 @@ class Kloud(object): self.multicast_mode = multicast_mode self.credentials = cred self.osclient_session = cred.get_session() - self.flavor_to_use = None self.vm_up_count = 0 self.res_logger = KBResLogger() + self.vm_img = vm_img if testing_side: self.prefix = 'KBc' self.name = 'Client Kloud' @@ -92,8 +101,43 @@ class Kloud(object): LOG.info("Creating kloud: " + self.prefix) if self.placement_az: LOG.info('%s Availability Zone: %s' % (self.name, self.placement_az)) + # A dict of flavors indexed by flavor name + self.flavors = {} + + def select_flavor(self): + # Select an existing flavor that Flavor check + flavor_manager = base_compute.Flavor(self.nova_client) + fcand = {'vcpus': sys.maxint, 'ram': sys.maxint, 'disk': sys.maxint} + # find the smallest flavor that is at least 1vcpu, 1024MB ram and 10MB disk + for flavor in flavor_manager.list(): + flavor = vars(flavor) + if flavor['vcpus'] < 1 or flavor['ram'] < 1024 or flavor['disk'] < 10: + continue + if flavor['vcpus'] < fcand['vcpus']: + fcand = flavor + elif flavor['vcpus'] == fcand['vcpus']: + if flavor['ram'] < fcand['ram']: + fcand = flavor + elif flavor['ram'] == fcand['ram'] and flavor['disk'] < fcand['disk']: + fcand = flavor + find_flag = True + + if find_flag: + LOG.info('Automatically selecting flavor %s to instantiate VMs.' % fcand['name']) + return fcand + LOG.error('Cannot find a flavor which meets the minimum ' + 'requirements to instantiate VMs.') + raise KBFlavorCheckException() def create_resources(self, tenant_quota): + def create_flavor(fm, name, flavor_dict, extra_specs): + flavor_dict['name'] = name + flv = fm.create_flavor(flavor_dict) + if extra_specs: + flv.set_keys(extra_specs) + self.flavors[name] = flv + self.res_logger.log('flavors', vars(flv)['name'], vars(flv)['id']) + if self.reusing_tenants: for tenant_info in self.reusing_tenants: tenant_name = tenant_info['name'] @@ -112,7 +156,17 @@ class Kloud(object): for tenant_instance in self.tenant_list: tenant_instance.create_resources() - if not self.reusing_tenants: + # Create/reuse flavors for this cloud + if self.reusing_tenants: + # If tenants are reused, we do not create new flavors but pick one + # existing that is good enough + flavor = self.select_flavor() + if self.testing_side: + self.flavors[FLAVOR_KB_PROXY] = flavor + self.flavors[FLAVOR_KB_CLIENT] = flavor + else: + self.flavors[FLAVOR_KB_SERVER] = flavor + else: # Create flavors for servers, clients, and kb-proxy nodes nova_client = self.tenant_list[0].user_list[0].nova_client flavor_manager = base_compute.Flavor(nova_client) @@ -125,35 +179,29 @@ class Kloud(object): else: flavor_dict['ephemeral'] = 0 if self.testing_side: - flv = flavor_manager.create_flavor('KB.proxy', override=True, - ram=2048, vcpus=1, disk=0, ephemeral=0) - self.res_logger.log('flavors', vars(flv)['name'], vars(flv)['id']) - flv = flavor_manager.create_flavor('KB.client', override=True, **flavor_dict) - self.res_logger.log('flavors', vars(flv)['name'], vars(flv)['id']) + proxy_flavor = { + "vcpus": 1, + "ram": 2048, + "disk": 0, + "ephemeral": 0 + } + create_flavor(flavor_manager, FLAVOR_KB_PROXY, proxy_flavor, extra_specs) + create_flavor(flavor_manager, FLAVOR_KB_CLIENT, flavor_dict, extra_specs) else: - flv = flavor_manager.create_flavor('KB.server', override=True, **flavor_dict) - self.res_logger.log('flavors', vars(flv)['name'], vars(flv)['id']) - if extra_specs: - flv.set_keys(extra_specs) + create_flavor(flavor_manager, FLAVOR_KB_SERVER, flavor_dict, extra_specs) def delete_resources(self): - # Deleting flavors created by KloudBuster - try: - nova_client = self.tenant_list[0].user_list[0].nova_client - except Exception: - # NOVA Client is not yet initialized, so skip cleaning up... - return True + + if not self.reusing_tenants: + for fn, flavor in self.flavors.iteritems(): + LOG.info('Deleting flavor %s', fn) + try: + flavor.delete() + except Exception as exc: + LOG.warning('Error deleting flavor %s: %s', fn, str(exc)) flag = True - if not self.reusing_tenants: - flavor_manager = base_compute.Flavor(nova_client) - if self.testing_side: - flavor_manager.delete_flavor('KB.client') - flavor_manager.delete_flavor('KB.proxy') - else: - flavor_manager.delete_flavor('KB.server') - for tnt in self.tenant_list: flag = flag & tnt.delete_resources() @@ -256,7 +304,6 @@ class KloudBuster(object): self.storage_mode = storage_mode self.multicast_mode = multicast_mode - if topology and tenants_list: self.topology = None LOG.warning("REUSING MODE: Topology configs will be ignored.") @@ -289,6 +336,8 @@ class KloudBuster(object): self.fp_logfile = None self.kloud = None self.testing_kloud = None + self.server_vm_img = None + self.client_vm_img = None def get_hypervisor_list(self, cred): ret_list = [] @@ -314,64 +363,79 @@ class KloudBuster(object): return ret_list - def check_and_upload_images(self, retry_count=150): + def check_and_upload_image(self, kloud_name, image_name, image_url, sess, retry_count): + '''Check a VM image and upload it if not found + ''' + glance_client = glanceclient.Client('2', session=sess) + try: + # Search for the image + img = glance_client.images.list(filters={'name': image_name}).next() + # image found + return img + except StopIteration: + sys.exc_clear() + + # Trying to upload image + LOG.info("KloudBuster VM Image is not found in %s, trying to upload it..." % kloud_name) + if not image_url: + LOG.error('Configuration file is missing a VM image pathname (vm_image_name)') + return None retry = 0 + try: + LOG.info("Uploading VM Image from %s..." % image_url) + with open(image_url) as f_image: + img = glance_client.images.create(name=image_name, + disk_format="qcow2", + container_format="bare", + visibility="public") + glance_client.images.upload(img.id, image_data=f_image) + # Check for the image in glance + while img.status in ['queued', 'saving'] and retry < retry_count: + img = glance_client.images.get(img.id) + retry += 1 + LOG.debug("Image not yet active, retrying %s of %s...", retry, retry_count) + time.sleep(2) + if img.status != 'active': + LOG.error("Image uploaded but too long to get to active state") + raise Exception("Image update active state timeout") + except glance_exception.HTTPForbidden: + LOG.error("Cannot upload image without admin access. Please make " + "sure the image is uploaded and is either public or owned by you.") + return None + except IOError as exc: + # catch the exception for file based errors. + LOG.error("Failed while uploading the image. Please make sure the " + "image at the specified location %s is correct: %s", + image_url, str(exc)) + return None + except keystoneauth1.exceptions.http.NotFound as exc: + LOG.error("Authentication error while uploading the image: " + str(exc)) + return None + except Exception: + LOG.error(traceback.format_exc()) + LOG.error("Failed while uploading the image: %s", str(exc)) + return None + return img + + def check_and_upload_images(self, retry_count=150): image_name = self.client_cfg.image_name image_url = self.client_cfg.vm_image_file - kloud_name_list = ['Server kloud', 'Client kloud'] - session_list = [self.server_cred.get_session(), - self.client_cred.get_session()] - for kloud, sess in zip(kloud_name_list, session_list): - glance_client = glanceclient.Client('2', session=sess) - try: - # Search for the image - img = glance_client.images.list(filters={'name': image_name}).next() - continue - except StopIteration: - sys.exc_clear() - - # Trying to upload images - LOG.info("KloudBuster VM Image is not found in %s, trying to upload it..." % kloud) - if not image_url: - LOG.error('Configuration file is missing a VM image pathname (vm_image_name)') - return False - retry = 0 - try: - LOG.info("Uploading VM Image from %s..." % image_url) - with open(image_url) as f_image: - img = glance_client.images.create(name=image_name, - disk_format="qcow2", - container_format="bare", - visibility="public") - glance_client.images.upload(img.id, image_data=f_image) - # Check for the image in glance - while img.status in ['queued', 'saving'] and retry < retry_count: - img = glance_client.images.get(img.id) - retry += 1 - LOG.debug("Image not yet active, retrying %s of %s...", retry, retry_count) - time.sleep(2) - if img.status != 'active': - LOG.error("Image uploaded but too long to get to active state") - raise Exception("Image update active state timeout") - except glance_exception.HTTPForbidden: - LOG.error("Cannot upload image without admin access. Please make " - "sure the image is uploaded and is either public or owned by you.") - return False - except IOError as exc: - # catch the exception for file based errors. - LOG.error("Failed while uploading the image. Please make sure the " - "image at the specified location %s is correct: %s", - image_url, str(exc)) - return False - except keystoneauth1.exceptions.http.NotFound as exc: - LOG.error("Authentication error while uploading the image: " + str(exc)) - return False - except Exception: - LOG.error(traceback.format_exc()) - LOG.error("Failed while uploading the image: %s", str(exc)) - return False - return True - return True + self.server_vm_img = self.check_and_upload_image('Server kloud', + image_name, + image_url, + self.server_cred.get_session(), + retry_count) + if self.server_vm_img is None: + return False + if self.client_cred == self.server_cred: + self.client_vm_img = self.server_vm_img + else: + self.client_vm_img = self.check_and_upload_image('Client kloud', + image_name, + image_url, + self.client_cred.get_session(), + retry_count) + return self.client_vm_img is not None def print_provision_info(self): """ @@ -408,8 +472,7 @@ class KloudBuster(object): for ins in server_list: ins.user_data['role'] = 'HTTP_Server' ins.user_data['http_server_configs'] = ins.config['http_server_configs'] - ins.boot_info['flavor_type'] = 'KB.server' if \ - not self.tenants_list['server'] else self.kloud.flavor_to_use + ins.boot_info['flavor_type'] = FLAVOR_KB_SERVER ins.boot_info['user_data'] = str(ins.user_data) elif test_mode == 'multicast': # Nuttcp tests over first /25 @@ -433,8 +496,7 @@ class KloudBuster(object): ins.user_data['multicast_listener_address_start'] = listener_addr_start ins.user_data['ntp_clocks'] = clocks ins.user_data['pktsizes'] = self.client_cfg.multicast_tool_configs.pktsizes - ins.boot_info['flavor_type'] = 'KB.server' if \ - not self.tenants_list['server'] else self.kloud.flavor_to_use + ins.boot_info['flavor_type'] = FLAVOR_KB_SERVER ins.boot_info['user_data'] = str(ins.user_data) def gen_client_user_data(self, test_mode): @@ -457,8 +519,7 @@ class KloudBuster(object): ins.user_data['target_shared_interface_ip'] = server_list[idx].shared_interface_ip if role == 'Multicast_Client': ins.user_data['ntp_clocks'] = clocks - ins.boot_info['flavor_type'] = 'KB.client' if \ - not self.tenants_list['client'] else self.testing_kloud.flavor_to_use + ins.boot_info['flavor_type'] = FLAVOR_KB_CLIENT ins.boot_info['user_data'] = str(ins.user_data) else: for idx, ins in enumerate(client_list): @@ -466,8 +527,7 @@ class KloudBuster(object): ins.user_data['vm_name'] = ins.vm_name ins.user_data['redis_server'] = self.kb_proxy.fixed_ip ins.user_data['redis_server_port'] = 6379 - ins.boot_info['flavor_type'] = 'KB.client' if \ - not self.tenants_list['client'] else self.testing_kloud.flavor_to_use + ins.boot_info['flavor_type'] = FLAVOR_KB_CLIENT ins.boot_info['user_data'] = str(ins.user_data) def gen_metadata(self): @@ -505,12 +565,15 @@ class KloudBuster(object): tenant_quota = self.calc_tenant_quota() if not self.storage_mode: self.kloud = Kloud(self.server_cfg, self.server_cred, self.tenants_list['server'], + self.server_vm_img, storage_mode=self.storage_mode, multicast_mode=self.multicast_mode) self.server_vm_create_thread = threading.Thread(target=self.kloud.create_vms, args=[vm_creation_concurrency]) self.server_vm_create_thread.daemon = True self.testing_kloud = Kloud(self.client_cfg, self.client_cred, - self.tenants_list['client'], testing_side=True, + self.tenants_list['client'], + self.client_vm_img, + testing_side=True, storage_mode=self.storage_mode, multicast_mode=self.multicast_mode) self.client_vm_create_thread = threading.Thread(target=self.testing_kloud.create_vms, @@ -528,8 +591,7 @@ class KloudBuster(object): self.kb_proxy.vm_name = 'KB-PROXY' self.kb_proxy.user_data['role'] = 'KB-PROXY' - self.kb_proxy.boot_info['flavor_type'] = 'KB.proxy' if \ - not self.tenants_list['client'] else self.testing_kloud.flavor_to_use + self.kb_proxy.boot_info['flavor_type'] = FLAVOR_KB_PROXY if self.topology: proxy_hyper = self.topology.clients_rack[0] self.kb_proxy.boot_info['avail_zone'] = \ @@ -661,6 +723,7 @@ class KloudBuster(object): self.fp_logfile = None def get_tenant_vm_count(self, config): + # this does not apply for storage mode! return (config['routers_per_tenant'] * config['networks_per_router'] * config['vms_per_network']) @@ -721,35 +784,50 @@ class KloudBuster(object): return [server_quota, client_quota] def calc_nova_quota(self): - total_vm = self.get_tenant_vm_count(self.server_cfg) server_quota = {} - server_quota['instances'] = total_vm - server_quota['cores'] = total_vm * self.server_cfg['flavor']['vcpus'] - server_quota['ram'] = total_vm * self.server_cfg['flavor']['ram'] - client_quota = {} - total_vm = self.get_tenant_vm_count(self.client_cfg) + if self.storage_mode: + # in case of storage, the number of VMs is to be taken from the + # the storage config + total_vm = self.client_cfg['storage_stage_configs']['vm_count'] + else: + total_vm = self.get_tenant_vm_count(self.server_cfg) + server_quota['instances'] = total_vm + server_quota['cores'] = total_vm * self.server_cfg['flavor']['vcpus'] + server_quota['ram'] = total_vm * self.server_cfg['flavor']['ram'] + LOG.info('Server tenant Nova quotas: instances=%d vcpus=%d ram=%dMB', + server_quota['instances'], + server_quota['cores'], + server_quota['ram']) + total_vm = self.get_tenant_vm_count(self.client_cfg) + + # add 1 for the proxy client_quota['instances'] = total_vm + 1 client_quota['cores'] = total_vm * self.client_cfg['flavor']['vcpus'] + 1 client_quota['ram'] = total_vm * self.client_cfg['flavor']['ram'] + 2048 - + LOG.info('Client tenant Nova quotas: instances=%d vcpus=%d ram=%dMB', + client_quota['instances'], + client_quota['cores'], + client_quota['ram']) return [server_quota, client_quota] def calc_cinder_quota(self): - total_vm = self.get_tenant_vm_count(self.server_cfg) - svr_disk = self.server_cfg['flavor']['disk'] + # Cinder quotas are only set for storage mode + # Since storage mode only uses client tenant + # Server tenant cinder quota is only used for non-storage case + # we can leave the server quota empty server_quota = {} - server_quota['gigabytes'] = total_vm * svr_disk \ - if svr_disk != 0 else -1 - server_quota['volumes'] = total_vm - total_vm = self.get_tenant_vm_count(self.client_cfg) - clt_disk = self.client_cfg['flavor']['disk'] + # Client tenant quota is based on the number of + # storage VMs and disk size per VM + # (note this is not the flavor disk size!) client_quota = {} - client_quota['gigabytes'] = total_vm * clt_disk + 20 \ - if clt_disk != 0 else -1 - client_quota['volumes'] = total_vm - + if self.storage_mode: + storage_cfg = self.client_cfg['storage_stage_configs'] + vm_count = storage_cfg['vm_count'] + client_quota['gigabytes'] = vm_count * storage_cfg['disk_size'] + client_quota['volumes'] = vm_count + LOG.info('Cinder quotas: volumes=%d storage=%dGB', vm_count, client_quota['gigabytes']) return [server_quota, client_quota] def calc_tenant_quota(self): diff --git a/kloudbuster/tenant.py b/kloudbuster/tenant.py index b5a0546..1633117 100644 --- a/kloudbuster/tenant.py +++ b/kloudbuster/tenant.py @@ -18,14 +18,10 @@ import base_storage from keystoneclient import exceptions as keystone_exception import log as logging -import sys import users LOG = logging.getLogger(__name__) -class KBFlavorCheckException(Exception): - pass - class KBQuotaCheckException(Exception): pass @@ -99,30 +95,6 @@ class Tenant(object): neutron_quota.update_quota(self.tenant_quota['neutron']) def check_quota(self): - # Flavor check - flavor_manager = base_compute.Flavor(self.kloud.nova_client) - find_flag = False - fcand = {'vcpus': sys.maxint, 'ram': sys.maxint, 'disk': sys.maxint} - for flavor in flavor_manager.list(): - flavor = vars(flavor) - if flavor['vcpus'] < 1 or flavor['ram'] < 1024 or flavor['disk'] < 10: - continue - if flavor['vcpus'] < fcand['vcpus']: - fcand = flavor - if flavor['vcpus'] == fcand['vcpus'] and flavor['ram'] < fcand['ram']: - fcand = flavor - if flavor['vcpus'] == fcand['vcpus'] and flavor['ram'] == fcand['ram'] and\ - flavor['disk'] < fcand['disk']: - fcand = flavor - find_flag = True - - if find_flag: - LOG.info('Automatically selects flavor %s to instantiate VMs.' % fcand['name']) - self.kloud.flavor_to_use = fcand['name'] - else: - LOG.error('Cannot find a flavor which meets the minimum ' - 'requirements to instantiate VMs.') - raise KBFlavorCheckException() # Nova/Cinder/Neutron quota check tenant_id = self.tenant_id