From 871cefc65f839f83cc646a0a385d51d4e582d9e3 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Fri, 28 Aug 2015 16:30:39 -0700 Subject: [PATCH] Handle ssh failures with reboot in launch-node.py New systemd based distros reboot so quickly that the ssh connection errors returning 255 (or -1 in python because signed integers). Ignore return codes of -1 when rebooting over ssh as a result. All other return codes will be propogated properly. Change-Id: I272f00e9e07f1ed04f2b97d0e1609c6e8d49caf3 --- launch/launch-node.py | 10 +++++++++- launch/sshclient.py | 8 +++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/launch/launch-node.py b/launch/launch-node.py index 96001a2c5f..2bd28b1c22 100755 --- a/launch/launch-node.py +++ b/launch/launch-node.py @@ -133,7 +133,15 @@ def bootstrap_server(server, admin_pass, key, cert, environment, name, "--certname %s" % (environment, puppetmaster, certname), error_ok=True) utils.interpret_puppet_exitcodes(rc, output) - ssh_client.ssh("reboot") + try: + ssh_client.ssh("reboot") + except Exception as e: + # Some init system kill the connection too fast after reboot. + # Deal with it by ignoring ssh errors when rebooting. + if e.rc == -1: + pass + else: + raise def build_server( diff --git a/launch/sshclient.py b/launch/sshclient.py index f02e8f8789..81afe4301f 100644 --- a/launch/sshclient.py +++ b/launch/sshclient.py @@ -23,6 +23,12 @@ import sys import paramiko +class SSHException(Exception): + def __init__(self, message, rc): + super(SSHException, self).__init__(message) + self.rc = rc + + class SSHClient(object): def __init__(self, ip, username, password=None, pkey=None): client = paramiko.SSHClient() @@ -41,7 +47,7 @@ class SSHClient(object): ret = stdout.channel.recv_exit_status() print stderr.read() if (not error_ok) and ret: - raise Exception("Unable to %s" % command) + raise SSHException("Unable to %s" % command, ret) return ret, output def scp(self, source, dest):