Consume os-client-config directly

In order for us to notice config changes in clouds.yaml, rather than
consuming it indirectly via shade, create the cloud config directly
and pass it in to the shade factory function.

There would not be a huge win in teaching shade to notice and update
itself at the moment, as the update processing happens in a different
place and there isn't really a great way to have the check ask the old
provider manager to see if its cloud config has changed.

Change-Id: Ie4cf44adefda637b5987dd9d39ec398786086720
Depends-On: Icdd9acede81bc5fba60d877194048e24a62c9e5d

nodepool/fakeprovider.py

@@ -68,7 +68,12 @@ def get_bad_client():
 FAKE_CLIENT = None
 
 
-def get_fake_client(**kwargs):
+def fake_get_one_cloud(cloud_config, cloud_kwargs):
+    cloud_kwargs['validate'] = False
+    return cloud_config.get_one_cloud(**cloud_kwargs)
+
+
+def get_fake_client(*args, **kwargs):
     global FAKE_CLIENT
     if FAKE_CLIENT is None:
         FAKE_CLIENT = FakeOpenStackCloud()

nodepool/nodepool.py

@@ -21,6 +21,7 @@ import apscheduler.scheduler
 import gear
 import json
 import logging
+import os_client_config
 import os.path
 import paramiko
 import Queue
@@ -59,6 +60,39 @@ DELETE_DELAY = 1 * MINS      # Delay before deleting a node that has completed
 DEFAULT_QEMU_IMAGE_COMPAT_OPTIONS = "--qemu-img-options 'compat=0.10'"
 
 
+def _cloudKwargsFromProvider(provider):
+    cloud_kwargs = {}
+    if 'region_name' in provider:
+        cloud_kwargs['region_name'] = provider['region-name']
+    if 'api_timeout' in provider:
+        cloud_kwargs['api_timeout'] = provider['api-timeout']
+    # These are named from back when we only talked to Nova. They're
+    # actually compute service related
+    if 'service_type' in provider:
+        cloud_kwargs['compute_service_type'] = provider['service-type']
+    if 'service_name' in provider:
+        cloud_kwargs['compute_service_name'] = provider['service-name']
+    if 'cloud' in provider:
+        cloud_kwargs['cloud'] = provider['cloud']
+
+    auth_kwargs = {}
+    for auth_key in ('username', 'password', 'auth-url'):
+        if auth_key in provider:
+            new_key = auth_key.replace('-', '_')
+            auth_kwargs[new_key] = provider[auth_key]
+
+    if 'project-id' in provider:
+        auth_kwargs['project_name'] = provider['project-id']
+
+    cloud_kwargs['auth'] = auth_kwargs
+    return cloud_kwargs
+
+
+def _get_one_cloud(cloud_config, cloud_kwargs):
+    '''This is a function to allow for overriding it in tests.'''
+    return cloud_config.get_one_cloud(**cloud_kwargs)
+
+
 class LaunchNodepoolException(Exception):
     statsd_key = 'error.nodepool'
 
@@ -1203,6 +1237,7 @@ class NodePool(threading.Thread):
     def loadConfig(self):
         self.log.debug("Loading configuration")
         config = yaml.load(open(self.configfile))
+        cloud_config = os_client_config.OpenStackConfig()
 
         newconfig = Config()
         newconfig.db = None
@@ -1249,13 +1284,9 @@ class NodePool(threading.Thread):
             p = Provider()
             p.name = provider['name']
             newconfig.providers[p.name] = p
-            p.username = provider.get('username')
+
-            p.password = provider.get('password')
+            cloud_kwargs = _cloudKwargsFromProvider(provider)
-            p.project_id = provider.get('project-id')
+            p.cloud_config = _get_one_cloud(cloud_config, cloud_kwargs)
-            p.auth_url = provider.get('auth-url')
-            p.cloud = provider.get('cloud')
-            p.service_type = provider.get('service-type')
-            p.service_name = provider.get('service-name')
             p.region_name = provider.get('region-name')
             p.max_servers = provider['max-servers']
             p.keypair = provider.get('keypair', None)
@@ -1394,13 +1425,7 @@ class NodePool(threading.Thread):
 
     def _managersEquiv(self, new_pm, old_pm):
         # Check if provider details have changed
-        if (new_pm.username != old_pm.provider.username or
+        if (new_pm.cloud_config != old_pm.provider.cloud_config or
-            new_pm.password != old_pm.provider.password or
-            new_pm.project_id != old_pm.provider.project_id or
-            new_pm.auth_url != old_pm.provider.auth_url or
-            new_pm.cloud != old_pm.provider.cloud or
-            new_pm.service_type != old_pm.provider.service_type or
-            new_pm.service_name != old_pm.provider.service_name or
             new_pm.max_servers != old_pm.provider.max_servers or
             new_pm.pool != old_pm.provider.pool or
             new_pm.image_type != old_pm.provider.image_type or

nodepool/provider_manager.py

@@ -283,31 +283,9 @@ class ProviderManager(TaskManager):
         self._cloud_metadata_read = True
 
     def _getClient(self):
-        kwargs = {}
+        return shade.OpenStackCloud(
-        if self.provider.region_name:
+            self.provider.cloud_config.name,
-            kwargs['region_name'] = self.provider.region_name
+            **self.provider.cloud_config.config)
-        if self.provider.api_timeout:
-            kwargs['api_timeout'] = self.provider.api_timeout
-        # These are named from back when we only talked to Nova. They're
-        # actually compute service related
-        if self.provider.service_type:
-            kwargs['compute_service_type'] = self.provider.service_type
-        if self.provider.service_name:
-            kwargs['compute_service_name'] = self.provider.service_name
-        if self.provider.cloud is not None:
-            kwargs['cloud'] = self.provider.cloud
-
-        auth_kwargs = {}
-        for auth_attr in ('username', 'password', 'auth_url'):
-            auth_val = getattr(self.provider, auth_attr)
-            if auth_val is not None:
-                auth_kwargs[auth_attr] = auth_val
-
-        if self.provider.project_id is not None:
-            auth_kwargs['project_name'] = self.provider.project_id
-
-        kwargs['auth'] = auth_kwargs
-        return shade.openstack_cloud(**kwargs)
 
     def runTask(self, task):
         try:

nodepool/tests/__init__.py

@@ -78,8 +78,12 @@ class BaseTestCase(testtools.TestCase, testresources.ResourcedTestCase):
         self.setUpFakes()
 
     def setUpFakes(self):
-        self.useFixture(fixtures.MonkeyPatch('shade.openstack_cloud',
+        self.useFixture(fixtures.MonkeyPatch(
-                                             fakeprovider.get_fake_client))
+            'nodepool.provider_manager.ProviderManager._getClient',
+            fakeprovider.get_fake_client))
+        self.useFixture(fixtures.MonkeyPatch(
+            'nodepool.nodepool._get_one_cloud',
+            fakeprovider.fake_get_one_cloud))
         self.useFixture(fixtures.MonkeyPatch('novaclient.client.Client',
                                              fakeprovider.FakeClient))
 

requirements.txt

@@ -15,8 +15,9 @@ python-keystoneclient
 python-novaclient>=2.21.0
 PyMySQL
 PrettyTable>=0.6,<0.8
-six>=1.7.0
 # shade has a looser requirement on six than nodepool, so install six first
+six>=1.7.0
+os-client-config>=1.2.0
 shade
 diskimage-builder
 voluptuous