Set usage of v3 clients, deal with endpoint URLs

Not all clients used their v3 version, so this patch adds
usage of them.

Endpoint URL addresses can have different format so the patch
edits parsing of them.

When a user exists, tempestconf will inform that a new user was not created,
but will not update it's password anymore. This can be done manually through
the openstack API.

Co-Authored-By: Arx Cruz <arxcruz@redhat.com>
Change-Id: I1f37c29fff0704f58d2179668e5f78ee459d2d61
This commit is contained in:
Martin Kopec 2017-08-25 21:27:21 +00:00 committed by Arx Cruz
parent f16daec76a
commit aac0c65661
5 changed files with 193 additions and 91 deletions

View File

@ -69,14 +69,25 @@ class Service(object):
class VersionedService(Service):
def get_versions(self):
body = self.do_get(self.service_url, top_level=True)
def get_versions(self, top_level=True):
body = self.do_get(self.service_url, top_level=top_level)
body = json.loads(body)
return self.deserialize_versions(body)
def deserialize_versions(self, body):
return map(lambda x: x['id'], body['versions'])
def no_port_cut_url(self):
# if there is no port defined, cut the url from version to the end
u = urllib3.util.parse_url(self.service_url)
url = self.service_url
if u.port is None:
found = re.findall(r'v\d', url)
if len(found) > 0:
index = url.index(found[0])
url = self.service_url[:index]
return (url, u.port is not None)
class ComputeService(VersionedService):
def get_extensions(self):
@ -84,9 +95,16 @@ class ComputeService(VersionedService):
body = json.loads(body)
return map(lambda x: x['alias'], body['extensions'])
def get_versions(self):
url, top_level = self.no_port_cut_url()
body = self.do_get(url, top_level=top_level)
body = json.loads(body)
return self.deserialize_versions(body)
class ImageService(VersionedService):
pass
def get_versions(self):
return super(ImageService, self).get_versions(top_level=False)
class NetworkService(VersionedService):
@ -102,19 +120,49 @@ class VolumeService(VersionedService):
body = json.loads(body)
return map(lambda x: x['alias'], body['extensions'])
def get_versions(self):
url, top_level = self.no_port_cut_url()
body = self.do_get(url, top_level=top_level)
body = json.loads(body)
return self.deserialize_versions(body)
class IdentityService(VersionedService):
def __init__(self, name, service_url, token, disable_ssl_validation):
super(VersionedService, self).__init__(
name, service_url, token, disable_ssl_validation)
version = ''
if 'v2' in self.service_url:
version = '/v2.0'
if 'v3' in self.service_url:
version = ''
url_parse = urlparse.urlparse(self.service_url)
self.service_url = '{}://{}{}'.format(
url_parse.scheme, url_parse.netloc, version)
def get_extensions(self):
if 'v2.0' in self.service_url:
if 'v2' in self.service_url:
body = self.do_get(self.service_url + '/extensions')
else:
body = self.do_get(self.service_url + '/v2.0/extensions')
body = json.loads(body)
return map(lambda x: x['alias'], body['extensions']['values'])
body = json.loads(body)
return map(lambda x: x['alias'], body['extensions']['values'])
# Keystone api changed in v3, the concept of extensions change. Right
# now, all the existin extensions are part of keystone core api, so,
# there's no longer the /extensions endpoint. The extensions that are
# stable, are enabled by default, the ones marked as experimental are
# disabled by default. Checking the tempest source, there's no test
# pointing to extensions endpoint, so I am very confident that this
# will not be an issue. If so, we need to list all the /OS-XYZ
# extensions to identify what is enabled or not. This would be a manual
# check every time keystone change, add or delete an extension, so I
# rather prefer to return empty here for now.
return []
def deserialize_versions(self, body):
return map(lambda x: x['id'], body['versions']['values'])
def get_versions(self):
return super(IdentityService, self).get_versions(top_level=False)
class ObjectStorageService(Service):
def get_extensions(self):
@ -177,6 +225,10 @@ def discover(auth_provider, region, object_store_discovery=True,
service_catalog = 'serviceCatalog'
public_url = 'publicURL'
identity_port = urlparse.urlparse(auth_provider.auth_url).port
if identity_port is None:
identity_port = ""
else:
identity_port = ":" + str(identity_port)
identity_version = urlparse.urlparse(auth_provider.auth_url).path
if api_version == 3:
service_catalog = 'catalog'
@ -193,7 +245,7 @@ def discover(auth_provider, region, object_store_discovery=True,
ep = entry['endpoints'][0]
if 'identity' in ep[public_url]:
services[name]['url'] = ep[public_url].replace(
"/identity", ":{0}{1}".format(
"/identity", "{0}{1}".format(
identity_port, identity_version))
else:
services[name]['url'] = ep[public_url]
@ -202,7 +254,7 @@ def discover(auth_provider, region, object_store_discovery=True,
disable_ssl_certificate_validation)
if name == 'object-store' and not object_store_discovery:
services[name]['extensions'] = []
elif 'v3' not in ep['publicURL']: # is not v3 url
elif 'v3' not in ep[public_url]: # is not v3 url
services[name]['extensions'] = service.get_extensions()
services[name]['versions'] = service.get_versions()
return services

View File

@ -48,7 +48,6 @@ import urllib2
import os_client_config
from oslo_config import cfg
from tempest.common import identity
from tempest.lib import auth
from tempest.lib import exceptions
from tempest.lib.services.compute import flavors_client
@ -60,7 +59,10 @@ from tempest.lib.services.identity.v2 import tenants_client
from tempest.lib.services.identity.v2 import users_client
from tempest.lib.services.identity.v3 \
import identity_client as identity_v3_client
from tempest.lib.services.identity.v3 import projects_client
from tempest.lib.services.identity.v3 import roles_client as roles_v3_client
from tempest.lib.services.identity.v3 import services_client as s_client
from tempest.lib.services.identity.v3 import users_client as users_v3_client
from tempest.lib.services.image.v2 import images_client
from tempest.lib.services.network import networks_client
from tempest.lib.services.volume.v2 import services_client
@ -147,14 +149,12 @@ def main():
conf.set(section, key, value, priority=True)
uri = conf.get("identity", "uri")
api_version = 2
v3_only = False
if "v3" in uri and v3_only:
api_version = 3
if "v3" in uri:
api_version = 3
conf.set("identity", "auth_version", "v3")
conf.set("identity", "uri", uri.replace("v3", "v2.0"), priority=True)
conf.set("identity", "uri_v3", uri)
else:
# TODO(arxcruz) make a check if v3 is enabled
conf.set("identity", "uri_v3", uri.replace("v2.0", "v3"))
if args.non_admin:
conf.set("auth", "admin_username", "")
@ -352,12 +352,40 @@ def set_cloud_config_values(conf, args):
'Could not load some identity options from cloud config file')
class ProjectsClient(object):
def __init__(self, auth, catalog_type, identity_region, endpoint_type,
identity_version, **default_params):
self.identity_version = identity_version
self.project_class = tenants_client.TenantsClient if \
self.identity_version == "v2" else projects_client.ProjectsClient
self.client = self.project_class(auth, catalog_type, identity_region,
endpoint_type, **default_params)
def get_project_by_name(self, project_name):
if self.identity_version == "v2":
projects = self.client.list_tenants()['tenants']
else:
projects = self.client.list_projects()['projects']
for project in projects:
if project['name'] == project_name:
return project
raise exceptions.NotFound(
'No such tenant/project (%s) in %s' % (project_name, projects))
def create_project(self, name, description):
if self.identity_version == "v2":
self.client.create_tenant(name=name, description=description)
else:
self.client.create_project(name=name, description=description)
class ClientManager(object):
"""Manager of various OpenStack API clients.
Connections to clients are created on-demand, i.e. the client tries to
connect to the server only when it's being requested.
"""
def get_credentials(self, conf, username, tenant_name, password,
identity_version='v2'):
creds_kwargs = {'username': username,
@ -404,6 +432,30 @@ class ClientManager(object):
else:
return "v2"
def set_users_client(self, auth, conf, endpoint_type, default_params):
users_class = users_client.UsersClient
if "v3" in self.identity_version:
users_class = users_v3_client.UsersClient
self.users = users_class(
auth,
conf.get_defaulted('identity', 'catalog_type'),
self.identity_region,
endpoint_type=endpoint_type,
**default_params)
def set_roles_client(self, auth, conf, endpoint_type, default_params):
roles_class = roles_client.RolesClient
if "v3" in self.identity_version:
roles_class = roles_v3_client.RolesClient
self.roles = roles_class(
auth,
conf.get_defaulted('identity', 'catalog_type'),
self.identity_region,
endpoint_type=endpoint_type,
**default_params)
def __init__(self, conf, admin):
self.identity_version = self.get_identity_version(conf)
username = None
@ -476,26 +528,25 @@ class ClientManager(object):
self.identity_region, endpoint_type='adminURL',
**default_params)
self.tenants = tenants_client.TenantsClient(
self.tenants = ProjectsClient(
_auth,
conf.get_defaulted('identity', 'catalog_type'),
self.identity_region,
endpoint_type='adminURL',
'adminURL',
self.identity_version,
**default_params)
self.roles = roles_client.RolesClient(
_auth,
conf.get_defaulted('identity', 'catalog_type'),
self.identity_region,
self.set_roles_client(
auth=_auth,
conf=conf,
endpoint_type='adminURL',
**default_params)
default_params=default_params)
self.users = users_client.UsersClient(
_auth,
conf.get_defaulted('identity', 'catalog_type'),
self.identity_region,
self.set_users_client(
auth=_auth,
conf=conf,
endpoint_type='adminURL',
**default_params)
default_params=default_params)
self.images = images_client.ImagesClient(
_auth,
@ -544,8 +595,7 @@ class ClientManager(object):
# Set admin tenant id needed for keystone v3 tests.
if admin:
tenant_id = identity.get_tenant_by_name(self.tenants,
tenant_name)['id']
tenant_id = self.tenants.get_project_by_name(tenant_name)['id']
conf.set('identity', 'admin_tenant_id', tenant_id)
@ -665,7 +715,7 @@ def create_tempest_users(tenants_client, roles_client, users_client, conf,
def give_role_to_user(tenants_client, roles_client, users_client, username,
tenant_name, role_name, role_required=True):
"""Give the user a role in the project (tenant).""",
tenant_id = identity.get_tenant_by_name(tenants_client, tenant_name)['id']
tenant_id = tenants_client.get_project_by_name(tenant_name)['id']
users = users_client.list_users()
user_ids = [u['id'] for u in users['users'] if u['name'] == username]
user_id = user_ids[0]
@ -688,32 +738,27 @@ def give_role_to_user(tenants_client, roles_client, users_client, username,
def create_user_with_tenant(tenants_client, users_client, username,
password, tenant_name):
"""Create user and tenant if he doesn't exist.
"""Create a user and a tenant if it doesn't exist."""
Sets password even for existing user.
"""
LOG.info("Creating user '%s' with tenant '%s' and password '%s'",
username, tenant_name, password)
tenant_description = "Tenant for Tempest %s user" % username
email = "%s@test.com" % username
# create tenant
# create a tenant
try:
tenants_client.create_tenant(name=tenant_name,
description=tenant_description)
tenants_client.create_project(name=tenant_name,
description=tenant_description)
except exceptions.Conflict:
LOG.info("(no change) Tenant '%s' already exists", tenant_name)
tenant_id = identity.get_tenant_by_name(tenants_client, tenant_name)['id']
# create user
tenant_id = tenants_client.get_project_by_name(tenant_name)['id']
# create a user
try:
users_client.create_user(**{'name': username, 'password': password,
'tenantId': tenant_id, 'email': email})
except exceptions.Conflict:
LOG.info("User '%s' already exists. Setting password to '%s'",
username, password)
user = identity.get_user_by_username(tenants_client, tenant_id,
username)
users_client.update_user_password(user['id'], password=password)
LOG.info("User '%s' already exists.", username)
def create_tempest_flavors(client, conf, allow_creation):
@ -943,7 +988,9 @@ def configure_boto(conf, services):
def configure_horizon(conf):
"""Derive the horizon URIs from the identity's URI."""
uri = conf.get('identity', 'uri')
base = uri.rsplit(':', 1)[0] + '/dashboard'
u = urllib2.urlparse.urlparse(uri)
host = u.netloc.split(":")[0]
base = '%s://%s%s' % (u.scheme, host, '/dashboard')
assert base.startswith('http:') or base.startswith('https:')
has_horizon = True
try:

View File

@ -126,7 +126,7 @@ class TestIdentityService(BaseServiceTest):
def setUp(self):
super(TestIdentityService, self).setUp()
self.Service = api.IdentityService("ServiceName",
self.FAKE_URL,
self.FAKE_URL + 'v2.0/',
self.FAKE_TOKEN,
disable_ssl_validation=False)

View File

@ -100,7 +100,8 @@ class TestClientManager(BaseConfigTempestTest):
def test_init_manager_as_admin(self):
mock_function = mock.Mock(return_value={"id": "my_fake_id"})
func2mock = 'config_tempest.config_tempest.identity.get_tenant_by_name'
func2mock = ('config_tempest.config_tempest.ProjectsClient.'
'get_project_by_name')
self.useFixture(MonkeyPatch(func2mock, mock_function))
self._get_clients(self.conf, admin=True)
# check if admin credentials were set
@ -117,7 +118,8 @@ class TestClientManager(BaseConfigTempestTest):
self.conf = self._get_alt_conf("v2.0", "v3")
self.client = self._get_clients(self.conf)
mock_function = mock.Mock(return_value={"id": "my_fake_id"})
func2mock = 'config_tempest.config_tempest.identity.get_tenant_by_name'
func2mock = ('config_tempest.config_tempest.ProjectsClient'
'.get_project_by_name')
self.useFixture(MonkeyPatch(func2mock, mock_function))
self._get_clients(self.conf, admin=True)
# check if admin credentials were set
@ -159,7 +161,8 @@ class TestOsClientConfigSupport(BaseConfigTempestTest):
func2mock = 'os_client_config.cloud_config.CloudConfig.config.get'
self.useFixture(MonkeyPatch(func2mock, mock_function))
mock_function = mock.Mock(return_value={"id": "my_fake_id"})
func2mock = 'config_tempest.config_tempest.identity.get_tenant_by_name'
func2mock = ('config_tempest.config_tempest.ProjectsClient.'
'get_project_by_name')
self.useFixture(MonkeyPatch(func2mock, mock_function))
def _obtain_client_config_data(self, mock_args, admin):

View File

@ -110,50 +110,49 @@ class TestCreateUserWithTenant(BaseConfigTempestTest):
self.tenant_description = "Tenant for Tempest %s user" % self.username
self.email = "%s@test.com" % self.username
@mock.patch('tempest.common.identity.get_tenant_by_name')
@mock.patch('tempest.lib.services.identity.v2.tenants_client.'
'TenantsClient.create_tenant')
@mock.patch('config_tempest.config_tempest.ProjectsClient'
'.get_project_by_name')
@mock.patch('config_tempest.config_tempest.ProjectsClient.create_project')
@mock.patch('tempest.lib.services.identity.v2.users_client.'
'UsersClient.create_user')
def test_create_user_with_tenant(self,
mock_create_user,
mock_create_tenant,
mock_get_tenant_by_name):
mock_get_tenant_by_name.return_value = {'id': "fake-id"}
mock_create_project,
mock_get_project_by_name):
mock_get_project_by_name.return_value = {'id': "fake-id"}
tool.create_user_with_tenant(
tenants_client=self.tenants_client,
users_client=self.users_client,
username=self.username,
password=self.password,
tenant_name=self.tenant_name)
mock_create_tenant.assert_called_with(
mock_create_project.assert_called_with(
name=self.tenant_name, description=self.tenant_description)
mock_create_user.assert_called_with(name=self.username,
password=self.password,
tenantId="fake-id",
email=self.email)
@mock.patch('tempest.common.identity.get_tenant_by_name')
@mock.patch(
'tempest.lib.services.identity.v2.'
'tenants_client.TenantsClient.create_tenant')
@mock.patch('config_tempest.config_tempest.ProjectsClient'
'.get_project_by_name')
@mock.patch('config_tempest.config_tempest.ProjectsClient.create_project')
@mock.patch('tempest.lib.services.identity.v2'
'.users_client.UsersClient.create_user')
def test_create_user_with_tenant_tenant_exists(
self,
mock_create_user,
mock_create_tenant,
mock_get_tenant_by_name):
mock_get_tenant_by_name.return_value = {'id': "fake-id"}
mock_create_project,
mock_get_project_by_name):
mock_get_project_by_name.return_value = {'id': "fake-id"}
exc = exceptions.Conflict
mock_create_tenant.side_effect = exc
mock_create_project.side_effect = exc
tool.create_user_with_tenant(
tenants_client=self.tenants_client,
users_client=self.users_client,
username=self.username,
password=self.password,
tenant_name=self.tenant_name)
mock_create_tenant.assert_called_with(
mock_create_project.assert_called_with(
name=self.tenant_name, description=self.tenant_description)
mock_create_user.assert_called_with(
name=self.username,
@ -164,17 +163,18 @@ class TestCreateUserWithTenant(BaseConfigTempestTest):
@mock.patch('tempest.lib.services.identity.v2.'
'users_client.UsersClient.update_user_password')
@mock.patch('tempest.common.identity.get_user_by_username')
@mock.patch('tempest.common.identity.get_tenant_by_name')
@mock.patch('config_tempest.config_tempest.ProjectsClient.'
'get_project_by_name')
@mock.patch('tempest.lib.services.identity.v2.'
'tenants_client.TenantsClient.create_tenant')
@mock.patch('tempest.lib.services.identity.'
'v2.users_client.UsersClient.create_user')
def test_create_user_with_tenant_user_exists(
self, mock_create_user, mock_create_tenant,
mock_get_tenant_by_name,
mock_get_project_by_name,
mock_get_user_by_username,
mock_update_user_password):
mock_get_tenant_by_name.return_value = {'id': "fake-id"}
mock_get_project_by_name.return_value = {'id': "fake-id"}
exc = exceptions.Conflict
mock_create_user.side_effect = exc
fake_user = {'id': "fake_user_id"}
@ -190,25 +190,23 @@ class TestCreateUserWithTenant(BaseConfigTempestTest):
password=self.password,
tenantId="fake-id",
email=self.email)
mock_update_user_password.assert_called_with(
fake_user['id'], password=self.password)
@mock.patch('tempest.lib.services.identity.v2.'
'users_client.UsersClient.update_user_password')
@mock.patch('tempest.common.identity.get_user_by_username')
@mock.patch('tempest.common.identity.get_tenant_by_name')
@mock.patch('tempest.lib.services.identity.v2.'
'tenants_client.TenantsClient.create_tenant')
@mock.patch('config_tempest.config_tempest.ProjectsClient.'
'get_project_by_name')
@mock.patch('config_tempest.config_tempest.ProjectsClient.create_project')
@mock.patch('tempest.lib.services.identity.v2.'
'users_client.UsersClient.create_user')
def test_create_user_with_tenant_exists_user_exists(
self, mock_create_user, mock_create_tenant,
mock_get_tenant_by_name,
self, mock_create_user, mock_create_project,
mock_get_project_by_name,
mock_get_user_by_username,
mock_update_user_password):
mock_get_tenant_by_name.return_value = {'id': "fake-id"}
mock_get_project_by_name.return_value = {'id': "fake-id"}
exc = exceptions.Conflict
mock_create_tenant.side_effects = exc
mock_create_project.side_effects = exc
mock_create_user.side_effect = exc
fake_user = {'id': "fake_user_id"}
mock_get_user_by_username.return_value = fake_user
@ -217,14 +215,12 @@ class TestCreateUserWithTenant(BaseConfigTempestTest):
username=self.username,
password=self.password,
tenant_name=self.tenant_name)
mock_create_tenant.assert_called_with(
mock_create_project.assert_called_with(
name=self.tenant_name, description=self.tenant_description)
mock_create_user.assert_called_with(name=self.username,
password=self.password,
tenantId="fake-id",
email=self.email)
mock_update_user_password.assert_called_with(
fake_user['id'], password=self.password)
class TestGiveRoleToUser(BaseConfigTempestTest):
@ -249,7 +245,8 @@ class TestGiveRoleToUser(BaseConfigTempestTest):
{'name': "fake_role2",
'id': "fake_role_id2"}]}
@mock.patch('tempest.common.identity.get_tenant_by_name')
@mock.patch('config_tempest.config_tempest.ProjectsClient.'
'get_project_by_name')
@mock.patch('tempest.lib.services.identity.v2.'
'users_client.UsersClient.list_users')
@mock.patch('tempest.lib.services.identity.v2.'
@ -263,9 +260,9 @@ class TestGiveRoleToUser(BaseConfigTempestTest):
mock_list_roles,
mock_create_user,
mock_list_users,
mock_get_tenant_by_name):
mock_get_project_by_name):
mock_get_tenant_by_name.return_value = \
mock_get_project_by_name.return_value = \
{'id': "fake_tenant_id"}
mock_list_users.return_value = self.users
mock_list_roles.return_value = self.roles
@ -279,7 +276,8 @@ class TestGiveRoleToUser(BaseConfigTempestTest):
mock_create_user_role_on_project.assert_called_with(
"fake_tenant_id", "fake_user_id", "fake_role_id")
@mock.patch('tempest.common.identity.get_tenant_by_name')
@mock.patch('config_tempest.config_tempest.ProjectsClient.'
'get_project_by_name')
@mock.patch('tempest.lib.services.identity.'
'v2.users_client.UsersClient.list_users')
@mock.patch('tempest.lib.services.identity.v2.'
@ -294,9 +292,9 @@ class TestGiveRoleToUser(BaseConfigTempestTest):
mock_list_roles,
mock_create_user,
mock_list_users,
mock_get_tenant_by_name):
mock_get_project_by_name):
role_name = "fake_role_that_does_not_exist"
mock_get_tenant_by_name.return_value = \
mock_get_project_by_name.return_value = \
{'id': "fake_tenant_id"}
mock_list_users.return_value = self.users
mock_list_roles.return_value = self.roles
@ -310,7 +308,8 @@ class TestGiveRoleToUser(BaseConfigTempestTest):
tenant_name=self.tenant_name,
role_name=role_name)
@mock.patch('tempest.common.identity.get_tenant_by_name')
@mock.patch('config_tempest.config_tempest.ProjectsClient.'
'get_project_by_name')
@mock.patch('tempest.lib.services.identity.v2.'
'users_client.UsersClient.list_users')
@mock.patch('tempest.lib.services.identity.v2.'
@ -325,9 +324,9 @@ class TestGiveRoleToUser(BaseConfigTempestTest):
mock_list_roles,
mock_create_user,
mock_list_users,
mock_get_tenant_by_name):
mock_get_project_by_name):
mock_get_tenant_by_name.return_value = \
mock_get_project_by_name.return_value = \
{'id': "fake_tenant_id"}
mock_list_users.return_value = self.users
mock_list_roles.return_value = self.roles
@ -340,7 +339,8 @@ class TestGiveRoleToUser(BaseConfigTempestTest):
role_name=self.role_name,
role_required=False)
@mock.patch('tempest.common.identity.get_tenant_by_name')
@mock.patch('config_tempest.config_tempest.ProjectsClient'
'.get_project_by_name')
@mock.patch('tempest.lib.services.identity.v2.'
'users_client.UsersClient.list_users')
@mock.patch('tempest.lib.services.identity.v2.'
@ -355,10 +355,10 @@ class TestGiveRoleToUser(BaseConfigTempestTest):
mock_list_roles,
mock_create_user,
mock_list_users,
mock_get_tenant_by_name):
mock_get_project_by_name):
exc = exceptions.Conflict
mock_create_user_role_on_project.side_effect = exc
mock_get_tenant_by_name.return_value = {'id': "fake_tenant_id"}
mock_get_project_by_name.return_value = {'id': "fake_tenant_id"}
mock_list_users.return_value = self.users
mock_list_roles.return_value = self.roles
tool.give_role_to_user(