Add host-key-checking option to openstack providers

In some cases nodepool-launcher uses public API to launch nodes, but
doesn't have access to the private networks of nodes it launches.
Rather then failing, expose an option for operators to disable
ssh-keyscan and allow nodes to become ready.

Change-Id: I764398aa21461ef44048e9e6565d2ee3e01aaaf8
Signed-off-by: Paul Belanger <pabelanger@redhat.com>

doc/source/configuration.rst

@@ -448,6 +448,7 @@ Example::
       networks:
         - some-network-name
       auto-floating-ip: False
+      host-key-checking: True
       labels:
         - name: trusty
           min-ram: 8192
@@ -505,6 +506,12 @@ Example::
     internal private network, set the option to False to save floating ip
     for cloud provider. The default value is True.
 
+  ``host-key-checking`` (bool)
+    Specify custom behavior of validation of SSH host keys.  When set to False,
+    nodepool-launcher will not ssh-keyscan nodes after they are booted. This
+    might be needed if nodepool-launcher and the nodes it launches are on
+    different networks.  The default value is True.
+
 .. _provider_diskimages:
 
 diskimages

nodepool/driver/openstack/config.py

@@ -66,6 +66,7 @@ class ProviderPool(ConfigValue):
             other.max_servers != self.max_servers or
             other.max_ram != self.max_ram or
             other.azs != self.azs or
+            other.host_key_checking != self.host_key_checking or
             other.networks != self.networks):
             return False
         return True
@@ -172,6 +173,7 @@ class OpenStackProviderConfig(ProviderConfig):
             pp.azs = pool.get('availability-zones')
             pp.networks = pool.get('networks', [])
             pp.auto_floating_ip = bool(pool.get('auto-floating-ip', True))
+            pp.host_key_checking = bool(pool.get('host-key-checking', True))
             pp.labels = {}
             for label in pool.get('labels', []):
                 pl = ProviderLabel()
@@ -253,6 +255,7 @@ class OpenStackProviderConfig(ProviderConfig):
             'name': str,
             'networks': [str],
             'auto-floating-ip': bool,
+            'host-key-checking': bool,
             'max-cores': int,
             'max-servers': int,
             'max-ram': int,

nodepool/driver/openstack/handler.py

@@ -195,16 +195,19 @@ class NodeLauncher(threading.Thread, stats.StatsReporter):
              self._node.public_ipv6))
 
         # Get the SSH public keys for the new node and record in ZooKeeper
-        try:
-            self.log.debug("Gathering host keys for node %s", self._node.id)
-            host_keys = utils.keyscan(
-                interface_ip, timeout=self._provider_config.boot_timeout)
-            if not host_keys:
-                raise exceptions.LaunchKeyscanException(
-                    "Unable to gather host keys")
-        except exceptions.SSHTimeoutException:
-            self.logConsole(self._node.external_id, self._node.hostname)
-            raise
+        host_keys = []
+        if self._pool.host_key_checking:
+            try:
+                self.log.debug(
+                    "Gathering host keys for node %s", self._node.id)
+                host_keys = utils.keyscan(
+                    interface_ip, timeout=self._provider_config.boot_timeout)
+                if not host_keys:
+                    raise exceptions.LaunchKeyscanException(
+                        "Unable to gather host keys")
+            except exceptions.SSHTimeoutException:
+                self.logConsole(self._node.external_id, self._node.hostname)
+                raise
 
         self._node.host_keys = host_keys
         self._zk.storeNode(self._node)

nodepool/tests/fixtures/config_validate/good.yaml

@@ -35,6 +35,7 @@ providers:
       - name: main
         max-servers: 184
         auto-floating-ip: True
+        host-key-checking: True
         labels:
           - name: trusty
             diskimage: trusty
@@ -66,6 +67,7 @@ providers:
       - name: main
         max-servers: 184
         auto-floating-ip: False
+        host-key-checking: False
         labels:
           - name: trusty
             diskimage: trusty

nodepool/tests/fixtures/node-host-key-checking.yaml

@@ -0,0 +1,50 @@
+elements-dir: .
+images-dir: '{images_dir}'
+build-log-dir: '{build_log_dir}'
+build-log-retention: 1
+
+zookeeper-servers:
+  - host: {zookeeper_host}
+    port: {zookeeper_port}
+    chroot: {zookeeper_chroot}
+
+labels:
+  - name: fake-label
+    min-ready: 1
+
+providers:
+  - name: fake-provider
+    cloud: fake
+    driver: fake
+    region-name: fake-region
+    rate: 0.0001
+    diskimages:
+      - name: fake-image
+        meta:
+          key: value
+          key2: value
+    pools:
+      - name: main
+        max-servers: 96
+        host-key-checking: False
+        availability-zones:
+          - az1
+        networks:
+          - net-name
+        labels:
+          - name: fake-label
+            diskimage: fake-image
+            min-ram: 8192
+            flavor-name: 'Fake'
+
+diskimages:
+  - name: fake-image
+    elements:
+      - fedora
+      - vm
+    release: 21
+    env-vars:
+      TMPDIR: /opt/dib_tmp
+      DIB_IMAGE_CACHE: /opt/dib_cache
+      DIB_CLOUD_IMAGES: http://download.fedoraproject.org/pub/fedora/linux/releases/test/21-Beta/Cloud/Images/x86_64/
+      BASE_IMAGE_FILE: Fedora-Cloud-Base-20141029-21_Beta.x86_64.qcow2

nodepool/tests/test_launcher.py

@@ -407,6 +407,23 @@ class TestLauncher(tests.DBTestCase):
         self.assertEqual(nodes[0].username, 'zuul')
         self.assertNotEqual(nodes[0].host_keys, [])
 
+    def test_node_host_key_checking_false(self):
+        """Test that an image and node are created"""
+        configfile = self.setup_config('node-host-key-checking.yaml')
+        pool = self.useNodepool(configfile, watermark_sleep=1)
+        self.useBuilder(configfile)
+        pool.start()
+        image = self.waitForImage('fake-provider', 'fake-image')
+        self.assertEqual(image.username, 'zuul')
+        nodes = self.waitForNodes('fake-label')
+
+        self.assertEqual(len(nodes), 1)
+        self.assertEqual(nodes[0].provider, 'fake-provider')
+        self.assertEqual(nodes[0].type, 'fake-label')
+        self.assertEqual(nodes[0].username, 'zuul')
+        # We have no host_keys because host-key-checking is False.
+        self.assertEqual(nodes[0].host_keys, [])
+
     def test_multiple_launcher(self):
         """Test that an image and node are created with 2 launchers"""
         # nodepool-builder needs access to both providers to upload images