diff --git a/config_tempest/api_discovery.py b/config_tempest/api_discovery.py index bea8d840..f7e6d232 100644 --- a/config_tempest/api_discovery.py +++ b/config_tempest/api_discovery.py @@ -69,14 +69,25 @@ class Service(object): class VersionedService(Service): - def get_versions(self): - body = self.do_get(self.service_url, top_level=True) + def get_versions(self, top_level=True): + body = self.do_get(self.service_url, top_level=top_level) body = json.loads(body) return self.deserialize_versions(body) def deserialize_versions(self, body): return map(lambda x: x['id'], body['versions']) + def no_port_cut_url(self): + # if there is no port defined, cut the url from version to the end + u = urllib3.util.parse_url(self.service_url) + url = self.service_url + if u.port is None: + found = re.findall(r'v\d', url) + if len(found) > 0: + index = url.index(found[0]) + url = self.service_url[:index] + return (url, u.port is not None) + class ComputeService(VersionedService): def get_extensions(self): @@ -84,9 +95,16 @@ class ComputeService(VersionedService): body = json.loads(body) return map(lambda x: x['alias'], body['extensions']) + def get_versions(self): + url, top_level = self.no_port_cut_url() + body = self.do_get(url, top_level=top_level) + body = json.loads(body) + return self.deserialize_versions(body) + class ImageService(VersionedService): - pass + def get_versions(self): + return super(ImageService, self).get_versions(top_level=False) class NetworkService(VersionedService): @@ -102,19 +120,49 @@ class VolumeService(VersionedService): body = json.loads(body) return map(lambda x: x['alias'], body['extensions']) + def get_versions(self): + url, top_level = self.no_port_cut_url() + body = self.do_get(url, top_level=top_level) + body = json.loads(body) + return self.deserialize_versions(body) + class IdentityService(VersionedService): + def __init__(self, name, service_url, token, disable_ssl_validation): + super(VersionedService, self).__init__( + name, service_url, token, disable_ssl_validation) + version = '' + if 'v2' in self.service_url: + version = '/v2.0' + if 'v3' in self.service_url: + version = '' + url_parse = urlparse.urlparse(self.service_url) + self.service_url = '{}://{}{}'.format( + url_parse.scheme, url_parse.netloc, version) + def get_extensions(self): - if 'v2.0' in self.service_url: + if 'v2' in self.service_url: body = self.do_get(self.service_url + '/extensions') - else: - body = self.do_get(self.service_url + '/v2.0/extensions') - body = json.loads(body) - return map(lambda x: x['alias'], body['extensions']['values']) + body = json.loads(body) + return map(lambda x: x['alias'], body['extensions']['values']) + # Keystone api changed in v3, the concept of extensions change. Right + # now, all the existin extensions are part of keystone core api, so, + # there's no longer the /extensions endpoint. The extensions that are + # stable, are enabled by default, the ones marked as experimental are + # disabled by default. Checking the tempest source, there's no test + # pointing to extensions endpoint, so I am very confident that this + # will not be an issue. If so, we need to list all the /OS-XYZ + # extensions to identify what is enabled or not. This would be a manual + # check every time keystone change, add or delete an extension, so I + # rather prefer to return empty here for now. + return [] def deserialize_versions(self, body): return map(lambda x: x['id'], body['versions']['values']) + def get_versions(self): + return super(IdentityService, self).get_versions(top_level=False) + class ObjectStorageService(Service): def get_extensions(self): @@ -177,6 +225,10 @@ def discover(auth_provider, region, object_store_discovery=True, service_catalog = 'serviceCatalog' public_url = 'publicURL' identity_port = urlparse.urlparse(auth_provider.auth_url).port + if identity_port is None: + identity_port = "" + else: + identity_port = ":" + str(identity_port) identity_version = urlparse.urlparse(auth_provider.auth_url).path if api_version == 3: service_catalog = 'catalog' @@ -193,7 +245,7 @@ def discover(auth_provider, region, object_store_discovery=True, ep = entry['endpoints'][0] if 'identity' in ep[public_url]: services[name]['url'] = ep[public_url].replace( - "/identity", ":{0}{1}".format( + "/identity", "{0}{1}".format( identity_port, identity_version)) else: services[name]['url'] = ep[public_url] @@ -202,7 +254,7 @@ def discover(auth_provider, region, object_store_discovery=True, disable_ssl_certificate_validation) if name == 'object-store' and not object_store_discovery: services[name]['extensions'] = [] - elif 'v3' not in ep['publicURL']: # is not v3 url + elif 'v3' not in ep[public_url]: # is not v3 url services[name]['extensions'] = service.get_extensions() services[name]['versions'] = service.get_versions() return services diff --git a/config_tempest/config_tempest.py b/config_tempest/config_tempest.py index eec9d883..51a86e24 100755 --- a/config_tempest/config_tempest.py +++ b/config_tempest/config_tempest.py @@ -48,7 +48,6 @@ import urllib2 import os_client_config from oslo_config import cfg -from tempest.common import identity from tempest.lib import auth from tempest.lib import exceptions from tempest.lib.services.compute import flavors_client @@ -60,7 +59,10 @@ from tempest.lib.services.identity.v2 import tenants_client from tempest.lib.services.identity.v2 import users_client from tempest.lib.services.identity.v3 \ import identity_client as identity_v3_client +from tempest.lib.services.identity.v3 import projects_client +from tempest.lib.services.identity.v3 import roles_client as roles_v3_client from tempest.lib.services.identity.v3 import services_client as s_client +from tempest.lib.services.identity.v3 import users_client as users_v3_client from tempest.lib.services.image.v2 import images_client from tempest.lib.services.network import networks_client from tempest.lib.services.volume.v2 import services_client @@ -147,14 +149,12 @@ def main(): conf.set(section, key, value, priority=True) uri = conf.get("identity", "uri") api_version = 2 - v3_only = False - if "v3" in uri and v3_only: - api_version = 3 if "v3" in uri: + api_version = 3 conf.set("identity", "auth_version", "v3") - conf.set("identity", "uri", uri.replace("v3", "v2.0"), priority=True) conf.set("identity", "uri_v3", uri) else: + # TODO(arxcruz) make a check if v3 is enabled conf.set("identity", "uri_v3", uri.replace("v2.0", "v3")) if args.non_admin: conf.set("auth", "admin_username", "") @@ -352,12 +352,40 @@ def set_cloud_config_values(conf, args): 'Could not load some identity options from cloud config file') +class ProjectsClient(object): + def __init__(self, auth, catalog_type, identity_region, endpoint_type, + identity_version, **default_params): + self.identity_version = identity_version + self.project_class = tenants_client.TenantsClient if \ + self.identity_version == "v2" else projects_client.ProjectsClient + self.client = self.project_class(auth, catalog_type, identity_region, + endpoint_type, **default_params) + + def get_project_by_name(self, project_name): + if self.identity_version == "v2": + projects = self.client.list_tenants()['tenants'] + else: + projects = self.client.list_projects()['projects'] + for project in projects: + if project['name'] == project_name: + return project + raise exceptions.NotFound( + 'No such tenant/project (%s) in %s' % (project_name, projects)) + + def create_project(self, name, description): + if self.identity_version == "v2": + self.client.create_tenant(name=name, description=description) + else: + self.client.create_project(name=name, description=description) + + class ClientManager(object): """Manager of various OpenStack API clients. Connections to clients are created on-demand, i.e. the client tries to connect to the server only when it's being requested. """ + def get_credentials(self, conf, username, tenant_name, password, identity_version='v2'): creds_kwargs = {'username': username, @@ -404,6 +432,30 @@ class ClientManager(object): else: return "v2" + def set_users_client(self, auth, conf, endpoint_type, default_params): + users_class = users_client.UsersClient + if "v3" in self.identity_version: + users_class = users_v3_client.UsersClient + + self.users = users_class( + auth, + conf.get_defaulted('identity', 'catalog_type'), + self.identity_region, + endpoint_type=endpoint_type, + **default_params) + + def set_roles_client(self, auth, conf, endpoint_type, default_params): + roles_class = roles_client.RolesClient + if "v3" in self.identity_version: + roles_class = roles_v3_client.RolesClient + + self.roles = roles_class( + auth, + conf.get_defaulted('identity', 'catalog_type'), + self.identity_region, + endpoint_type=endpoint_type, + **default_params) + def __init__(self, conf, admin): self.identity_version = self.get_identity_version(conf) username = None @@ -476,26 +528,25 @@ class ClientManager(object): self.identity_region, endpoint_type='adminURL', **default_params) - self.tenants = tenants_client.TenantsClient( + self.tenants = ProjectsClient( _auth, conf.get_defaulted('identity', 'catalog_type'), self.identity_region, - endpoint_type='adminURL', + 'adminURL', + self.identity_version, **default_params) - self.roles = roles_client.RolesClient( - _auth, - conf.get_defaulted('identity', 'catalog_type'), - self.identity_region, + self.set_roles_client( + auth=_auth, + conf=conf, endpoint_type='adminURL', - **default_params) + default_params=default_params) - self.users = users_client.UsersClient( - _auth, - conf.get_defaulted('identity', 'catalog_type'), - self.identity_region, + self.set_users_client( + auth=_auth, + conf=conf, endpoint_type='adminURL', - **default_params) + default_params=default_params) self.images = images_client.ImagesClient( _auth, @@ -544,8 +595,7 @@ class ClientManager(object): # Set admin tenant id needed for keystone v3 tests. if admin: - tenant_id = identity.get_tenant_by_name(self.tenants, - tenant_name)['id'] + tenant_id = self.tenants.get_project_by_name(tenant_name)['id'] conf.set('identity', 'admin_tenant_id', tenant_id) @@ -665,7 +715,7 @@ def create_tempest_users(tenants_client, roles_client, users_client, conf, def give_role_to_user(tenants_client, roles_client, users_client, username, tenant_name, role_name, role_required=True): """Give the user a role in the project (tenant).""", - tenant_id = identity.get_tenant_by_name(tenants_client, tenant_name)['id'] + tenant_id = tenants_client.get_project_by_name(tenant_name)['id'] users = users_client.list_users() user_ids = [u['id'] for u in users['users'] if u['name'] == username] user_id = user_ids[0] @@ -688,32 +738,27 @@ def give_role_to_user(tenants_client, roles_client, users_client, username, def create_user_with_tenant(tenants_client, users_client, username, password, tenant_name): - """Create user and tenant if he doesn't exist. + """Create a user and a tenant if it doesn't exist.""" - Sets password even for existing user. - """ LOG.info("Creating user '%s' with tenant '%s' and password '%s'", username, tenant_name, password) tenant_description = "Tenant for Tempest %s user" % username email = "%s@test.com" % username - # create tenant + # create a tenant try: - tenants_client.create_tenant(name=tenant_name, - description=tenant_description) + tenants_client.create_project(name=tenant_name, + description=tenant_description) except exceptions.Conflict: LOG.info("(no change) Tenant '%s' already exists", tenant_name) - tenant_id = identity.get_tenant_by_name(tenants_client, tenant_name)['id'] - # create user + tenant_id = tenants_client.get_project_by_name(tenant_name)['id'] + + # create a user try: users_client.create_user(**{'name': username, 'password': password, 'tenantId': tenant_id, 'email': email}) except exceptions.Conflict: - LOG.info("User '%s' already exists. Setting password to '%s'", - username, password) - user = identity.get_user_by_username(tenants_client, tenant_id, - username) - users_client.update_user_password(user['id'], password=password) + LOG.info("User '%s' already exists.", username) def create_tempest_flavors(client, conf, allow_creation): @@ -943,7 +988,9 @@ def configure_boto(conf, services): def configure_horizon(conf): """Derive the horizon URIs from the identity's URI.""" uri = conf.get('identity', 'uri') - base = uri.rsplit(':', 1)[0] + '/dashboard' + u = urllib2.urlparse.urlparse(uri) + host = u.netloc.split(":")[0] + base = '%s://%s%s' % (u.scheme, host, '/dashboard') assert base.startswith('http:') or base.startswith('https:') has_horizon = True try: diff --git a/config_tempest/tests/test_api_discovery_services.py b/config_tempest/tests/test_api_discovery_services.py index 0d05dd80..74ce9c0b 100644 --- a/config_tempest/tests/test_api_discovery_services.py +++ b/config_tempest/tests/test_api_discovery_services.py @@ -126,7 +126,7 @@ class TestIdentityService(BaseServiceTest): def setUp(self): super(TestIdentityService, self).setUp() self.Service = api.IdentityService("ServiceName", - self.FAKE_URL, + self.FAKE_URL + 'v2.0/', self.FAKE_TOKEN, disable_ssl_validation=False) diff --git a/config_tempest/tests/test_config_tempest.py b/config_tempest/tests/test_config_tempest.py index ddf1949a..c01ed64e 100644 --- a/config_tempest/tests/test_config_tempest.py +++ b/config_tempest/tests/test_config_tempest.py @@ -100,7 +100,8 @@ class TestClientManager(BaseConfigTempestTest): def test_init_manager_as_admin(self): mock_function = mock.Mock(return_value={"id": "my_fake_id"}) - func2mock = 'config_tempest.config_tempest.identity.get_tenant_by_name' + func2mock = ('config_tempest.config_tempest.ProjectsClient.' + 'get_project_by_name') self.useFixture(MonkeyPatch(func2mock, mock_function)) self._get_clients(self.conf, admin=True) # check if admin credentials were set @@ -117,7 +118,8 @@ class TestClientManager(BaseConfigTempestTest): self.conf = self._get_alt_conf("v2.0", "v3") self.client = self._get_clients(self.conf) mock_function = mock.Mock(return_value={"id": "my_fake_id"}) - func2mock = 'config_tempest.config_tempest.identity.get_tenant_by_name' + func2mock = ('config_tempest.config_tempest.ProjectsClient' + '.get_project_by_name') self.useFixture(MonkeyPatch(func2mock, mock_function)) self._get_clients(self.conf, admin=True) # check if admin credentials were set @@ -159,7 +161,8 @@ class TestOsClientConfigSupport(BaseConfigTempestTest): func2mock = 'os_client_config.cloud_config.CloudConfig.config.get' self.useFixture(MonkeyPatch(func2mock, mock_function)) mock_function = mock.Mock(return_value={"id": "my_fake_id"}) - func2mock = 'config_tempest.config_tempest.identity.get_tenant_by_name' + func2mock = ('config_tempest.config_tempest.ProjectsClient.' + 'get_project_by_name') self.useFixture(MonkeyPatch(func2mock, mock_function)) def _obtain_client_config_data(self, mock_args, admin): diff --git a/config_tempest/tests/test_config_tempest_user.py b/config_tempest/tests/test_config_tempest_user.py index da61fe52..60510b69 100644 --- a/config_tempest/tests/test_config_tempest_user.py +++ b/config_tempest/tests/test_config_tempest_user.py @@ -110,50 +110,49 @@ class TestCreateUserWithTenant(BaseConfigTempestTest): self.tenant_description = "Tenant for Tempest %s user" % self.username self.email = "%s@test.com" % self.username - @mock.patch('tempest.common.identity.get_tenant_by_name') - @mock.patch('tempest.lib.services.identity.v2.tenants_client.' - 'TenantsClient.create_tenant') + @mock.patch('config_tempest.config_tempest.ProjectsClient' + '.get_project_by_name') + @mock.patch('config_tempest.config_tempest.ProjectsClient.create_project') @mock.patch('tempest.lib.services.identity.v2.users_client.' 'UsersClient.create_user') def test_create_user_with_tenant(self, mock_create_user, - mock_create_tenant, - mock_get_tenant_by_name): - mock_get_tenant_by_name.return_value = {'id': "fake-id"} + mock_create_project, + mock_get_project_by_name): + mock_get_project_by_name.return_value = {'id': "fake-id"} tool.create_user_with_tenant( tenants_client=self.tenants_client, users_client=self.users_client, username=self.username, password=self.password, tenant_name=self.tenant_name) - mock_create_tenant.assert_called_with( + mock_create_project.assert_called_with( name=self.tenant_name, description=self.tenant_description) mock_create_user.assert_called_with(name=self.username, password=self.password, tenantId="fake-id", email=self.email) - @mock.patch('tempest.common.identity.get_tenant_by_name') - @mock.patch( - 'tempest.lib.services.identity.v2.' - 'tenants_client.TenantsClient.create_tenant') + @mock.patch('config_tempest.config_tempest.ProjectsClient' + '.get_project_by_name') + @mock.patch('config_tempest.config_tempest.ProjectsClient.create_project') @mock.patch('tempest.lib.services.identity.v2' '.users_client.UsersClient.create_user') def test_create_user_with_tenant_tenant_exists( self, mock_create_user, - mock_create_tenant, - mock_get_tenant_by_name): - mock_get_tenant_by_name.return_value = {'id': "fake-id"} + mock_create_project, + mock_get_project_by_name): + mock_get_project_by_name.return_value = {'id': "fake-id"} exc = exceptions.Conflict - mock_create_tenant.side_effect = exc + mock_create_project.side_effect = exc tool.create_user_with_tenant( tenants_client=self.tenants_client, users_client=self.users_client, username=self.username, password=self.password, tenant_name=self.tenant_name) - mock_create_tenant.assert_called_with( + mock_create_project.assert_called_with( name=self.tenant_name, description=self.tenant_description) mock_create_user.assert_called_with( name=self.username, @@ -164,17 +163,18 @@ class TestCreateUserWithTenant(BaseConfigTempestTest): @mock.patch('tempest.lib.services.identity.v2.' 'users_client.UsersClient.update_user_password') @mock.patch('tempest.common.identity.get_user_by_username') - @mock.patch('tempest.common.identity.get_tenant_by_name') + @mock.patch('config_tempest.config_tempest.ProjectsClient.' + 'get_project_by_name') @mock.patch('tempest.lib.services.identity.v2.' 'tenants_client.TenantsClient.create_tenant') @mock.patch('tempest.lib.services.identity.' 'v2.users_client.UsersClient.create_user') def test_create_user_with_tenant_user_exists( self, mock_create_user, mock_create_tenant, - mock_get_tenant_by_name, + mock_get_project_by_name, mock_get_user_by_username, mock_update_user_password): - mock_get_tenant_by_name.return_value = {'id': "fake-id"} + mock_get_project_by_name.return_value = {'id': "fake-id"} exc = exceptions.Conflict mock_create_user.side_effect = exc fake_user = {'id': "fake_user_id"} @@ -190,25 +190,23 @@ class TestCreateUserWithTenant(BaseConfigTempestTest): password=self.password, tenantId="fake-id", email=self.email) - mock_update_user_password.assert_called_with( - fake_user['id'], password=self.password) @mock.patch('tempest.lib.services.identity.v2.' 'users_client.UsersClient.update_user_password') @mock.patch('tempest.common.identity.get_user_by_username') - @mock.patch('tempest.common.identity.get_tenant_by_name') - @mock.patch('tempest.lib.services.identity.v2.' - 'tenants_client.TenantsClient.create_tenant') + @mock.patch('config_tempest.config_tempest.ProjectsClient.' + 'get_project_by_name') + @mock.patch('config_tempest.config_tempest.ProjectsClient.create_project') @mock.patch('tempest.lib.services.identity.v2.' 'users_client.UsersClient.create_user') def test_create_user_with_tenant_exists_user_exists( - self, mock_create_user, mock_create_tenant, - mock_get_tenant_by_name, + self, mock_create_user, mock_create_project, + mock_get_project_by_name, mock_get_user_by_username, mock_update_user_password): - mock_get_tenant_by_name.return_value = {'id': "fake-id"} + mock_get_project_by_name.return_value = {'id': "fake-id"} exc = exceptions.Conflict - mock_create_tenant.side_effects = exc + mock_create_project.side_effects = exc mock_create_user.side_effect = exc fake_user = {'id': "fake_user_id"} mock_get_user_by_username.return_value = fake_user @@ -217,14 +215,12 @@ class TestCreateUserWithTenant(BaseConfigTempestTest): username=self.username, password=self.password, tenant_name=self.tenant_name) - mock_create_tenant.assert_called_with( + mock_create_project.assert_called_with( name=self.tenant_name, description=self.tenant_description) mock_create_user.assert_called_with(name=self.username, password=self.password, tenantId="fake-id", email=self.email) - mock_update_user_password.assert_called_with( - fake_user['id'], password=self.password) class TestGiveRoleToUser(BaseConfigTempestTest): @@ -249,7 +245,8 @@ class TestGiveRoleToUser(BaseConfigTempestTest): {'name': "fake_role2", 'id': "fake_role_id2"}]} - @mock.patch('tempest.common.identity.get_tenant_by_name') + @mock.patch('config_tempest.config_tempest.ProjectsClient.' + 'get_project_by_name') @mock.patch('tempest.lib.services.identity.v2.' 'users_client.UsersClient.list_users') @mock.patch('tempest.lib.services.identity.v2.' @@ -263,9 +260,9 @@ class TestGiveRoleToUser(BaseConfigTempestTest): mock_list_roles, mock_create_user, mock_list_users, - mock_get_tenant_by_name): + mock_get_project_by_name): - mock_get_tenant_by_name.return_value = \ + mock_get_project_by_name.return_value = \ {'id': "fake_tenant_id"} mock_list_users.return_value = self.users mock_list_roles.return_value = self.roles @@ -279,7 +276,8 @@ class TestGiveRoleToUser(BaseConfigTempestTest): mock_create_user_role_on_project.assert_called_with( "fake_tenant_id", "fake_user_id", "fake_role_id") - @mock.patch('tempest.common.identity.get_tenant_by_name') + @mock.patch('config_tempest.config_tempest.ProjectsClient.' + 'get_project_by_name') @mock.patch('tempest.lib.services.identity.' 'v2.users_client.UsersClient.list_users') @mock.patch('tempest.lib.services.identity.v2.' @@ -294,9 +292,9 @@ class TestGiveRoleToUser(BaseConfigTempestTest): mock_list_roles, mock_create_user, mock_list_users, - mock_get_tenant_by_name): + mock_get_project_by_name): role_name = "fake_role_that_does_not_exist" - mock_get_tenant_by_name.return_value = \ + mock_get_project_by_name.return_value = \ {'id': "fake_tenant_id"} mock_list_users.return_value = self.users mock_list_roles.return_value = self.roles @@ -310,7 +308,8 @@ class TestGiveRoleToUser(BaseConfigTempestTest): tenant_name=self.tenant_name, role_name=role_name) - @mock.patch('tempest.common.identity.get_tenant_by_name') + @mock.patch('config_tempest.config_tempest.ProjectsClient.' + 'get_project_by_name') @mock.patch('tempest.lib.services.identity.v2.' 'users_client.UsersClient.list_users') @mock.patch('tempest.lib.services.identity.v2.' @@ -325,9 +324,9 @@ class TestGiveRoleToUser(BaseConfigTempestTest): mock_list_roles, mock_create_user, mock_list_users, - mock_get_tenant_by_name): + mock_get_project_by_name): - mock_get_tenant_by_name.return_value = \ + mock_get_project_by_name.return_value = \ {'id': "fake_tenant_id"} mock_list_users.return_value = self.users mock_list_roles.return_value = self.roles @@ -340,7 +339,8 @@ class TestGiveRoleToUser(BaseConfigTempestTest): role_name=self.role_name, role_required=False) - @mock.patch('tempest.common.identity.get_tenant_by_name') + @mock.patch('config_tempest.config_tempest.ProjectsClient' + '.get_project_by_name') @mock.patch('tempest.lib.services.identity.v2.' 'users_client.UsersClient.list_users') @mock.patch('tempest.lib.services.identity.v2.' @@ -355,10 +355,10 @@ class TestGiveRoleToUser(BaseConfigTempestTest): mock_list_roles, mock_create_user, mock_list_users, - mock_get_tenant_by_name): + mock_get_project_by_name): exc = exceptions.Conflict mock_create_user_role_on_project.side_effect = exc - mock_get_tenant_by_name.return_value = {'id': "fake_tenant_id"} + mock_get_project_by_name.return_value = {'id': "fake_tenant_id"} mock_list_users.return_value = self.users mock_list_roles.return_value = self.roles tool.give_role_to_user(