x/kloudbuster
Code Issues Proposed changes

Enhance the supports for tenant/user reusing

Browse Source 
When running under tenant/user reusing mode:
1. Adding features to automatically pick the flavor;
2. Adding features to check quotas before testing;
3. Merge the server and client tenant into one;

Change-Id: Ibdda7624174056d7770de02ed0813cad6db2ac05
This commit is contained in:
Yichen Wang 2015-07-23 12:18:03 -07:00
parent edefeac214
commit 65f0580c41
7 changed files with 129 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
scale/base_compute.py
View File

@ -217,6 +217,9 @@ class Flavor(object):
def __init__(self, novaclient):
self.novaclient = novaclient
def list(self):
return self.novaclient.flavors.list()
def create_flavor(self, name, ram, vcpus, disk, override=False):
# Creating flavors
if override:
@ -236,6 +239,9 @@ class NovaQuota(object):
self.novaclient = novaclient
self.tenant_id = tenant_id
def get(self):
return self.novaclient.quotas.get(self.tenant_id).__dict__
def update_quota(self, **kwargs):
self.novaclient.quotas.update(self.tenant_id, **kwargs)
@ -245,5 +251,8 @@ class CinderQuota(object):
self.cinderclient = cinderclient
self.tenant_id = tenant_id
def get(self):
return self.cinderclient.quotas.get(self.tenant_id).__dict__
def update_quota(self, **kwargs):
self.cinderclient.quotas.update(self.tenant_id, **kwargs)

11
scale/base_network.py
View File

@ -23,12 +23,14 @@ from neutronclient.common.exceptions import NetworkInUseClient
LOG = logging.getLogger(__name__)
# Global CIDR shared by all objects of this class
# Enables each network to get a unique CIDR
START_CIDR = "10.0.0.0/16"
cidr = START_CIDR
class KBGetExtNetException(Exception):
pass
def create_floating_ip(neutron_client, ext_net):
"""
Function that creates a floating ip and returns it
@ -63,8 +65,8 @@ def find_external_network(neutron_client):
if network['router:external']:
return network
LOG.error("No external network found!!!")
return None
LOG.error("No external network is found.")
raise KBGetExtNetException()
class BaseNetwork(object):
@ -375,6 +377,9 @@ class NeutronQuota(object):
self.neutronclient = neutronclient
self.tenant_id = tenant_id
def get(self):
return self.neutronclient.show_quota(self.tenant_id)['quota']
def update_quota(self, quotas):
body = {
'quota': quotas

34
scale/cfg.tenants.yaml
View File

@ -7,22 +7,20 @@
#
# Settings in this file has higher priority than user configs. It determines
# the final count of tenants and useres that KloudBuster will use.
#
# If running under tenant/user reusing mode, KloudBuster will use *only* one
# tenant to hold the resources for both server cloud and client cloud.
#
# NOTE:
# (1) For now, we only support one user per tenant;
# (2) Under tenant/user resuing mode, all resources will be sitting under
# the same tenant, so there will be fixed *ONLY* one user for holding
# client side resources;
# Sections for listing the tenant and user lists for server cloud.
# Note: For now we only support 1 user per tenant
server:
- name: ts_1
user:
- username: tsu_1
password: tsu_1
- name: ts_2
user:
- username: tsu_2
password: tsu_2
client:
- name: tc_1
user:
- username: tcu_1
password: tcu_1
tenant_name: demo_tenant
server_user:
- username: demo_user_1
password: demo_user_1
client_user:
username: demo_user_2
password: demo_user_2

4
scale/kb_config.py
View File

@ -154,8 +154,8 @@ class KBConfig(object):
if CONF.tenants_list:
self.tenants_list = configure.Configuration.from_file(CONF.tenants_list).configure()
try:
self.config_scale['number_tenants'] = len(self.tenants_list['server'])
self.config_scale['users_per_tenant'] = len(self.tenants_list['server'][0]['user'])
self.config_scale['number_tenants'] = 1
self.config_scale['users_per_tenant'] = len(self.tenants_list['server_user'])
except Exception as e:
LOG.error('Cannot parse the count of tenant/user from the config file.')
raise KBConfigParseException(e.message)

18
scale/kloudbuster.py
View File

@ -65,8 +65,8 @@ def check_and_upload_images(cred, cred_testing, server_img_name, client_img_name
if img['name'] == img_name_dict[kloud]:
img_found = True
break
if img.visibility != 'public' and CONF.tenants_list:
LOG.error("Image must be public when running in reusing mode.")
if img_found and img.visibility != 'public' and CONF.tenants_list:
LOG.error("Image must be public when running in tenant/user reusing mode.")
sys.exit(1)
if not img_found:
@ -236,9 +236,13 @@ class KloudBuster(object):
else:
self.topology = topology
if tenants_list:
self.tenants_list = tenants_list
LOG.warn("REUSING MODE: The quota will not adjust automatically.")
LOG.warn("REUSING MODE: The flavor configs will be ignored, and m1.small is used.")
self.tenants_list = {}
self.tenants_list['server'] =\
[{'name': tenants_list['tenant_name'], 'user': tenants_list['server_user']}]
self.tenants_list['client'] =\
[{'name': tenants_list['tenant_name'], 'user': [tenants_list['client_user']]}]
LOG.warn("REUSING MODE: The quotas will not be adjusted automatically.")
LOG.warn("REUSING MODE: The flavor configs will be ignored.")
else:
self.tenants_list = {'server': None, 'client': None}
# TODO(check on same auth_url instead)
@ -427,7 +431,7 @@ class KloudBuster(object):
server_quota['floatingip'] = server_quota['router']
server_quota['port'] = total_vm + 2 * server_quota['network'] + server_quota['router']
server_quota['security_group'] = server_quota['network'] + 1
server_quota['security_group_rule'] = server_quota['security_group'] * 100
server_quota['security_group_rule'] = server_quota['security_group'] * 10
client_quota = {}
total_vm = total_vm * self.server_cfg['number_tenants']
@ -454,7 +458,7 @@ class KloudBuster(object):
# cloud, and each one takes up 1 port on client side.
client_quota['port'] = client_quota['port'] + server_quota['router']
client_quota['security_group'] = client_quota['network'] + 1
client_quota['security_group_rule'] = client_quota['security_group'] * 100
client_quota['security_group_rule'] = client_quota['security_group'] * 10
return [server_quota, client_quota]

36
scale/tenant.py
View File

@ -34,23 +34,28 @@ class Tenant(object):
Stores the shared network in case of testing and
tested cloud being on same cloud
"""
self.tenant_name = tenant_name
self.kloud = kloud
self.tenant_object = self._get_tenant()
self.tenant_id = self.tenant_object.id
self.tenant_name = tenant_name
if not self.kloud.reusing_tenants:
self.tenant_object = self._get_tenant()
self.tenant_id = self.tenant_object.id
else:
LOG.info("Using tenant: " + self.tenant_name)
# Only admin can retrive the object via Keystone API
self.tenant_object = None
try:
# Try to see if we have the admin access to retrive the tenant id using
# tenant name directly from keystone
self.tenant_id = self.kloud.keystone.tenants.find(name=self.tenant_name).id
except Exception:
self.tenant_id = self.kloud.keystone.tenant_id
self.tenant_quota = tenant_quota
self.reusing_users = reusing_users
# Contains a list of user instance objects
self.user_list = []
def _get_tenant(self):
if self.kloud.reusing_tenants:
LOG.info("Using tenant: " + self.tenant_name)
tenant_list = self.kloud.keystone.tenants.list()
for tenant in tenant_list:
if tenant.name == self.tenant_name:
return tenant
raise Exception("Tenant not found")
'''
Create or reuse a tenant object of a given name
@ -68,15 +73,10 @@ class Tenant(object):
if exc.http_status != 409:
raise exc
LOG.info("Tenant %s already present, reusing it" % self.tenant_name)
# It is a hassle to find a tenant by name as the only way seems to retrieve
# the list of all tenants which can be very large
tenant_list = self.kloud.keystone.tenants.list()
for tenant in tenant_list:
if tenant.name == self.tenant_name:
return tenant
return self.kloud.keystone.tenants.find(name=self.tenant_name)
# Should never come here
raise Exception("Tenant not found")
raise Exception()
def create_resources(self):
"""
@ -87,7 +87,7 @@ class Tenant(object):
for user_info in self.reusing_users:
user_name = user_info['username']
password = user_info['password']
user_instance = users.User(user_name, password, self, None)
user_instance = users.User(user_name, password, self, '_member_')
self.user_list.append(user_instance)
else:
# Loop over the required number of users and create resources

95
scale/users.py
View File

@ -22,6 +22,11 @@ from novaclient.client import Client
LOG = logging.getLogger(__name__)
class KBFlavorCheckException(Exception):
pass
class KBQuotaCheckException(Exception):
pass
class User(object):
"""
@ -45,7 +50,6 @@ class User(object):
self.nova_client = None
self.neutron_client = None
self.cinder_client = None
self.user = self._get_user()
# Each user is associated to 1 key pair at most
self.key_pair = None
self.key_name = None
@ -56,15 +60,17 @@ class User(object):
#
# If running on top of existing tenants/users, skip
# the step for admin role association.
if not self.tenant.kloud.reusing_tenants:
current_role = None
for role in self.tenant.kloud.keystone.roles.list():
if role.name == user_role:
current_role = role
break
if not self.tenant.reusing_users:
self.user = self._get_user()
current_role = self.tenant.kloud.keystone.roles.find(name=user_role)
self.tenant.kloud.keystone.roles.add_user_role(self.user,
current_role,
tenant.tenant_id)
else:
# Only admin can retrive the object via Keystone API
self.user = None
LOG.info("Using user: " + self.user_name)
def _create_user(self):
LOG.info("Creating user: " + self.user_name)
@ -74,14 +80,6 @@ class User(object):
tenant_id=self.tenant.tenant_id)
def _get_user(self):
if self.tenant.reusing_users:
LOG.info("Using user: " + self.user_name)
users_list = self.tenant.kloud.keystone.users.list()
for user in users_list:
if user.name == self.user_name:
return user
raise Exception('Cannot find stale user:' + self.user_name)
'''
Create a new user or reuse if it already exists (on a different tenant)
delete the user and create a new one
@ -97,19 +95,13 @@ class User(object):
# Try to repair keystone by removing that user
LOG.warn("User creation failed due to stale user with same name: " +
self.user_name)
# Again, trying to find a user by name is pretty inefficient as one has to list all
# of them
users_list = self.tenant.kloud.keystone.users.list()
for user in users_list:
if user.name == self.user_name:
# Found it, time to delete it
LOG.info("Deleting stale user with name: " + self.user_name)
self.tenant.kloud.keystone.users.delete(user)
user = self._create_user()
return user
user = self.tenant.kloud.keystone.users.find(name=self.user_name)
LOG.info("Deleting stale user with name: " + self.user_name)
self.tenant.kloud.keystone.users.delete(user)
return self._create_user()
# Not found there is something wrong
raise Exception('Cannot find stale user:' + self.user_name)
# Should never come here
raise Exception()
def delete_resources(self):
LOG.info("Deleting all user resources for user %s" % self.user_name)
@ -136,6 +128,47 @@ class User(object):
neutron_quota = base_network.NeutronQuota(self.neutron_client, self.tenant.tenant_id)
neutron_quota.update_quota(tenant_quota['neutron'])
def check_resources_quota(self):
# Flavor check
flavor_manager = base_compute.Flavor(self.nova_client)
flavor_to_use = None
for flavor in flavor_manager.list():
flavor = flavor.__dict__
if flavor['vcpus'] < 1 or flavor['ram'] < 1024 or flavor['disk'] < 10:
continue
flavor_to_use = flavor
break
if flavor_to_use:
LOG.info('Automatically selects flavor %s to instantiate VMs.' %
(flavor_to_use['name']))
else:
LOG.error('Cannot find a flavor which meets the minimum '
'requirements to instantiate VMs.')
raise KBFlavorCheckException()
# Nova/Cinder/Neutron quota check
tenant_id = self.tenant.tenant_id
meet_quota = True
for quota_type in ['nova', 'cinder', 'neutron']:
if quota_type == 'nova':
quota_manager = base_compute.NovaQuota(self.nova_client, tenant_id)
elif quota_type == 'cinder':
quota_manager = base_compute.CinderQuota(self.cinder_client, tenant_id)
else:
quota_manager = base_network.NeutronQuota(self.neutron_client, tenant_id)
meet_quota = True
quota = quota_manager.get()
for key, value in self.tenant.tenant_quota[quota_type].iteritems():
if quota[key] < value:
meet_quota = False
break
if not meet_quota:
LOG.error('%s quota is too small. Minimum requirement: %s.' %
(quota_type, self.tenant.tenant_quota[quota_type]))
raise KBQuotaCheckException()
def create_resources(self):
"""
Creates all the User elements associated with a User
@ -145,7 +178,7 @@ class User(object):
# Create a new neutron client for this User with correct credentials
creden = {}
creden['username'] = self.user_name
creden['password'] = self.user_name
creden['password'] = self.password
creden['auth_url'] = self.tenant.kloud.auth_url
creden['tenant_name'] = self.tenant.tenant_name
@ -155,7 +188,7 @@ class User(object):
# Create a new nova and cinder client for this User with correct credentials
creden_nova = {}
creden_nova['username'] = self.user_name
creden_nova['api_key'] = self.user_name
creden_nova['api_key'] = self.password
creden_nova['auth_url'] = self.tenant.kloud.auth_url
creden_nova['project_id'] = self.tenant.tenant_name
creden_nova['version'] = 2
@ -163,7 +196,9 @@ class User(object):
self.nova_client = Client(**creden_nova)
self.cinder_client = cinderclient.Client(**creden_nova)
if not self.tenant.kloud.reusing_tenants:
if self.tenant.kloud.reusing_tenants:
self.check_resources_quota()
else:
self.update_tenant_quota(self.tenant.tenant_quota)
config_scale = self.tenant.kloud.scale_cfg