Remove static class loader

This commit remove the static class loader and add a dynamic one.
This is done using a new property available_classes that checks for
classes inherited by Services and generate a list of it.
It also change how tempestconf identify available services using the
list_services api instead of get_catalog api from tempest lib.
This way, tempestconf are no longer generating Services objects for
services that doesn't have a specialized class, but still set the
defaults for true or false if a service is running or not.
The discover method has changed to get the services from the list of
services instead of from the catalog.

Change-Id: Ic9d482378e9c975ed00f6aa7ed5f7aed6956d229
Story: 2002787
Task: 22672

config_tempest/services/base.py

@@ -73,6 +73,15 @@ class Service(object):
     def get_extensions(self):
         return self.extensions
 
+    @staticmethod
+    def get_service_name():
+        """Return the service name.
+
+        This return a list because you can have different services for the
+        same type, like volume, volumev2, volumev3
+        """
+        return []
+
     def get_versions(self):
         """Return the versions available for each service.
 

config_tempest/services/boto.py

@@ -21,8 +21,16 @@ class Ec2Service(Service):
     def set_default_tempest_options(self, conf):
         conf.set('boto', 'ec2_url', self.service_url)
 
+    @staticmethod
+    def get_service_name():
+        return ['ec2']
+
 
 class S3Service(Service):
 
     def set_default_tempest_options(self, conf):
         conf.set('boto', 's3_url', self.service_url)
+
+    @staticmethod
+    def get_service_name():
+        return ['s3']

config_tempest/services/compute.py

@@ -56,3 +56,7 @@ class ComputeService(VersionedService):
         except exceptions.Forbidden:
             C.LOG.info('Can not retrieve hosts, user are not allowed')
             return 1
+
+    @staticmethod
+    def get_service_name():
+        return ['nova']

config_tempest/services/identity.py

@@ -60,6 +60,10 @@ class IdentityService(VersionedService):
     def get_supported_versions(self):
         return ['v2', 'v3']
 
+    @staticmethod
+    def get_service_name():
+        return ['keystone']
+
     def get_catalog(self):
         return 'identity'
 

config_tempest/services/image.py

@@ -65,6 +65,10 @@ class ImageService(VersionedService):
     def get_supported_versions(self):
         return ['v1', 'v2']
 
+    @staticmethod
+    def get_service_name():
+        return ['glance']
+
     def get_catalog(self):
         return 'image'
 

config_tempest/services/network.py

@@ -91,3 +91,7 @@ class NetworkService(VersionedService):
             else:
                 raise Exception('fixed_network_name could not be '
                                 'discovered and must be specified')
+
+    @staticmethod
+    def get_service_name():
+        return ['neutron']

config_tempest/services/object_storage.py

@@ -105,3 +105,7 @@ class ObjectStorageService(Service):
         # Set roles based on service status
         if swift_status:
             self.list_create_roles(conf, self.client.roles)
+
+    @staticmethod
+    def get_service_name():
+        return ['swift']

config_tempest/services/octavia.py

@@ -26,3 +26,7 @@ class LoadBalancerService(VersionedService):
         conf.set('load_balancer', 'member_role', '_member_')
         conf.set('load_balancer', 'admin_role', 'admin')
         conf.set('load_balancer', 'RBAC_test_type', 'owner_or_admin')
+
+    @staticmethod
+    def get_service_name():
+        return ['octavia']

config_tempest/services/services.py

@@ -14,31 +14,19 @@
 # under the License.
 
 
+import importlib
+import pkgutil
+import pyclbr
+
 from six.moves import urllib
 
 from config_tempest import constants as C
-from config_tempest.services.base import Service
-from config_tempest.services import boto
 from config_tempest.services import ceilometer
-from config_tempest.services.compute import ComputeService
 from config_tempest.services import horizon
-from config_tempest.services.identity import IdentityService
-from config_tempest.services.image import ImageService
-from config_tempest.services.network import NetworkService
-from config_tempest.services.object_storage import ObjectStorageService
-from config_tempest.services.octavia import LoadBalancerService
 from config_tempest.services import volume
+from tempest.lib import exceptions
 
-
-service_dict = {'compute': ComputeService,
-                'image': ImageService,
-                'network': NetworkService,
-                'object-store': ObjectStorageService,
-                'volumev3': volume.VolumeService,
-                'identity': IdentityService,
-                'ec2': boto.Ec2Service,
-                's3': boto.S3Service,
-                'load-balancer': LoadBalancerService}
+import config_tempest.services
 
 
 class Services(object):
@@ -49,30 +37,82 @@ class Services(object):
         self._ssl_validation = creds.disable_ssl_certificate_validation
         self._region = clients.identity_region
         self._services = []
+        self._service_classes = []
         self.set_catalog_and_url()
+        self.available_services = self.get_available_services()
 
         self.discover()
 
+    @property
+    def service_classes(self):
+        """Return the list of classes available under config_tempest.services.
+
+        This return the list of classes that inherit from base.Service
+        """
+        if not self._service_classes:
+            path = config_tempest.services.__path__
+            prefix = config_tempest.services.__name__ + '.'
+            for importer, modname, ispkg in pkgutil.walk_packages(
+                    path=path, prefix=prefix, onerror=lambda x: None):
+                module_info = pyclbr.readmodule(modname)
+                for item in module_info.values():
+                    m = importlib.import_module(modname)
+                    c = getattr(m, item.name)
+                    if issubclass(c, config_tempest.services.base.Service):
+                        self._service_classes.append(c)
+
+        return self._service_classes
+
+    def get_available_services(self):
+        try:
+            services = self._clients.service_client.list_services()['services']
+            return {s['name']: s['type'] for s in services}
+        except exceptions.Forbidden:
+            C.LOG.warning("User has no permissions to list services, using "
+                          "catalog. Services without endpoint will not be "
+                          "discovered.")
+            token, auth_data = self._clients.auth_provider.get_auth()
+            return {s['name']: s['type'] for s
+                    in auth_data[self.service_catalog]}
+
     def discover(self):
         token, auth_data = self._clients.auth_provider.get_auth()
 
-        for entry in auth_data[self.service_catalog]:
-            name = entry['type']
-            url = self.parse_endpoints(self.get_endpoints(entry), name)
+        auth_entries = {e['type']: e for e in auth_data[self.service_catalog]}
 
-            service_class = self.get_service_class(name)
-            service = service_class(name, url, token, self._ssl_validation,
-                                    self._clients.get_service_client(name))
-            # discover extensions of the service
-            service.set_extensions()
-            # discover versions of the service
-            service.set_versions()
-            self.merge_exts_multiversion_service(service)
+        # We loop through the classes we have for each service, and if we find
+        # a class that match a service enabled, we add it in our services list.
+        # some services doesn't have endpoints, so we need to check first
+        for s_class in self.service_classes:
+            s_names = s_class.get_service_name()
+            for s_name in s_names:
+                s_type = self.available_services.get(s_name, None)
+                if s_type:
+                    endpoint_data = auth_entries.get(s_type, None)
+                    url = None
+                    if not endpoint_data:
+                        C.LOG.Warning('No endpoint data found for {}'.format(
+                            s_name))
+                    else:
+                        url = self.parse_endpoints(self.get_endpoints(
+                            endpoint_data), s_type)
 
-            # default tempest options
-            service.set_default_tempest_options(self._conf)
+                    # Create the service class and add it to services list
+                    service = s_class(s_type, url, token,
+                                      self._ssl_validation,
+                                      self._clients.get_service_client(
+                                          s_type))
+                    # discover extensions of the service
+                    service.set_extensions()
 
-            self._services.append(service)
+                    # discover versions of the service
+                    service.set_versions()
+                    self.merge_exts_multiversion_service(service)
+
+                    # default tempest options
+                    service.set_default_tempest_options(self._conf)
+
+                    self._services.append(service)
 
     def merge_exts_multiversion_service(self, service):
         """Merges extensions of a service given by its name
@@ -150,16 +190,6 @@ class Services(object):
         replace_text = port + "/identity/" + self._creds.identity_version
         return url.replace("/identity", replace_text)
 
-    def get_service_class(self, name):
-        """Returns class name by the service name
-
-        :param name: Codename of a service
-        :type name: string
-        :return: Class name of the service
-        :rtype: string
-        """
-        return service_dict.get(name, Service)
-
     def get_service(self, name):
         """Finds and returns a service object
 
@@ -179,7 +209,7 @@ class Services(object):
         :type name: string
         :rtype: boolean
         """
-        if self.get_service(name) is None:
+        if name not in self.available_services.values():
             return False
         return True
 
@@ -246,7 +276,8 @@ class Services(object):
         """
         extensions = []
         for o in service_objects:
-            extensions += o.extensions
+            if o:
+                extensions += o.extensions
         return extensions
 
     def set_service_extensions(self):

config_tempest/services/volume.py

@@ -63,6 +63,10 @@ class VolumeService(VersionedService):
     def get_unversioned_service_name(self):
         return 'volume'
 
+    @staticmethod
+    def get_service_name():
+        return ['cinderv2', 'cinderv3']
+
 
 def check_volume_backup_service(conf, volume_client, is_volumev3):
     """Verify if the cinder backup service is enabled"""

config_tempest/tests/services/test_boto.py

@@ -16,6 +16,8 @@
 import logging
 import mock
 
+from config_tempest.services.boto import Ec2Service
+from config_tempest.services.boto import S3Service
 from config_tempest.services.services import Services
 from config_tempest.tests.base import BaseConfigTempestTest
 
@@ -28,15 +30,16 @@ class TestEc2Service(BaseConfigTempestTest):
     FAKE_URL = "http://10.200.16.10:8774/"
 
     @mock.patch('config_tempest.services.services.Services.discover')
-    def setUp(self, mock_discover):
+    @mock.patch('config_tempest.services.services.Services.'
+                'get_available_services')
+    def setUp(self, mock_set_avail, mock_discover):
         super(TestEc2Service, self).setUp()
         conf = self._get_conf('v2', 'v3')
         self.clients = self._get_clients(conf)
         self.Services = Services(self.clients, conf, self._get_creds(conf))
 
     def test_set_default_tempest_options(self):
-        service_class = self.Services.get_service_class("ec2")
-        service = service_class("ec2", self.FAKE_URL, self.clients, False)
+        service = Ec2Service("ec2", self.FAKE_URL, self.clients, False)
         service.set_default_tempest_options(self.Services._conf)
         ec2_url = self.Services._conf.get("boto", "ec2_url")
         self.assertEqual(ec2_url, self.FAKE_URL)
@@ -47,15 +50,16 @@ class TestS3Service(BaseConfigTempestTest):
     FAKE_URL = "http://10.200.16.10:8774/"
 
     @mock.patch('config_tempest.services.services.Services.discover')
-    def setUp(self, mock_discover):
+    @mock.patch('config_tempest.services.services.Services.'
+                'get_available_services')
+    def setUp(self, mock_set_avail, mock_discover):
         super(TestS3Service, self).setUp()
         conf = self._get_conf('v2', 'v3')
         self.clients = self._get_clients(conf)
         self.Services = Services(self.clients, conf, self._get_creds(conf))
 
     def test_set_default_tempest_options(self):
-        service_class = self.Services.get_service_class("s3")
-        service = service_class("s3", self.FAKE_URL, self.clients, False)
+        service = S3Service("s3", self.FAKE_URL, self.clients, False)
         service.set_default_tempest_options(self.Services._conf)
         ec2_url = self.Services._conf.get("boto", "s3_url")
         self.assertEqual(ec2_url, self.FAKE_URL)

config_tempest/tests/services/test_services.py

@@ -37,7 +37,10 @@ class TestServices(BaseConfigTempestTest):
         super(TestServices, self).setUp()
 
     @mock.patch('config_tempest.services.services.Services.discover')
-    def _create_services_instance(self, mock_discover, v2=False):
+    @mock.patch('config_tempest.services.services.Services.'
+                'get_available_services')
+    def _create_services_instance(self, mock_avail, mock_discover, v2=False):
+        mock_avail.return_value = {'my_service': 'my_service'}
         conf = self._get_conf('v2', 'v3')
         creds = self._get_creds(conf, v2=v2)
         clients = mock.Mock()