diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 149c3edd6..90c3f195f 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -278,26 +278,32 @@ providers: diskimage: centos-7 min-ram: 1024 flavor-name: 'nodepool' + console-log: True - name: debian-jessie diskimage: debian-jessie min-ram: 512 flavor-name: 'nodepool' + console-log: True - name: fedora-25 diskimage: fedora-25 min-ram: 1024 flavor-name: 'nodepool' + console-log: True - name: ubuntu-precise diskimage: ubuntu-precise min-ram: 512 flavor-name: 'nodepool' + console-log: True - name: ubuntu-trusty diskimage: ubuntu-trusty min-ram: 512 flavor-name: 'nodepool' + console-log: True - name: ubuntu-xenial diskimage: ubuntu-xenial min-ram: 512 flavor-name: 'nodepool' + console-log: True diskimages: - name: centos-7 diff --git a/doc/source/configuration.rst b/doc/source/configuration.rst index a9ab6be82..6926bc286 100644 --- a/doc/source/configuration.rst +++ b/doc/source/configuration.rst @@ -237,6 +237,7 @@ Example:: - name: trusty min-ram: 8192 diskimage: trusty + console-log: True - name: precise min-ram: 8192 diskimage: precise @@ -356,6 +357,7 @@ Example:: - name: trusty min-ram: 8192 diskimage: trusty + console-log: True - name: precise min-ram: 8192 diskimage: precise @@ -475,6 +477,7 @@ Example configuration:: - name: precise min-ram: 8192 flavor-name: 'something to match' + console-log: True **required** @@ -509,3 +512,7 @@ Example configuration:: ``key-name`` If given, is the name of a keypair that will be used when booting each server. + + ``console-log`` (default: False) + On the failure of the ssh ready check, download the server console log to + aid in debuging the problem. diff --git a/nodepool/config.py b/nodepool/config.py index b0118a974..55d0a6d32 100644 --- a/nodepool/config.py +++ b/nodepool/config.py @@ -270,6 +270,8 @@ def loadConfig(config_path): pl.min_ram = label.get('min-ram', 0) pl.flavor_name = label.get('flavor-name', None) pl.key_name = label.get('key-name') + pl.console_log = label.get('console-log', False) + top_label = newconfig.labels[pl.name] top_label.pools.append(pp) diff --git a/nodepool/launcher.py b/nodepool/launcher.py index 73d016203..770c9d829 100644 --- a/nodepool/launcher.py +++ b/nodepool/launcher.py @@ -255,6 +255,15 @@ class NodeLauncher(threading.Thread, StatsReporter): self._diskimage = None self._cloud_image = self._provider.cloud_images.get(self._label.cloud_image, None) + def logConsole(self, server_id, hostname): + if not self._label.console_log: + return + console = self._manager.getServerConsole(server_id) + if console: + self.log.debug('Console log from hostname %s:' % hostname) + for line in console.splitlines(): + self.log.debug(line.rstrip()) + def _launchNode(self): if self._label.diskimage: # launch using diskimage @@ -359,11 +368,16 @@ class NodeLauncher(threading.Thread, StatsReporter): self._node.public_ipv4, self._node.public_ipv6)) # Get the SSH public keys for the new node and record in ZooKeeper - self.log.debug("Gathering host keys for node %s", self._node.id) - host_keys = utils.keyscan( - interface_ip, timeout=self._provider.boot_timeout) - if not host_keys: - raise LaunchKeyscanException("Unable to gather host keys") + try: + self.log.debug("Gathering host keys for node %s", self._node.id) + host_keys = utils.keyscan( + interface_ip, timeout=self._provider.boot_timeout) + if not host_keys: + raise LaunchKeyscanException("Unable to gather host keys") + except exceptions.SSHTimeoutException: + self.logConsole(self._node.external_id, self._node.hostname) + raise + self._node.host_keys = host_keys self._zk.storeNode(self._node) diff --git a/nodepool/provider_manager.py b/nodepool/provider_manager.py index 4e1241dd5..dc57507c7 100644 --- a/nodepool/provider_manager.py +++ b/nodepool/provider_manager.py @@ -238,6 +238,13 @@ class ProviderManager(object): with shade_inner_exceptions(): return self._client.get_server(server_id) + def getServerConsole(self, server_id): + try: + with shade_inner_exceptions(): + return self._client.get_server_console(server_id) + except shade.OpenStackCloudException: + return None + def waitForServer(self, server, timeout=3600): with shade_inner_exceptions(): return self._client.wait_for_server(