Handle ssh failures with reboot in launch-node.py

New systemd based distros reboot so quickly that the ssh connection
errors returning 255 (or -1 in python because signed integers). Ignore
return codes of -1 when rebooting over ssh as a result. All other return
codes will be propogated properly.

Change-Id: I272f00e9e07f1ed04f2b97d0e1609c6e8d49caf3

launch/launch-node.py

@@ -133,7 +133,15 @@ def bootstrap_server(server, admin_pass, key, cert, environment, name,
         "--certname %s" % (environment, puppetmaster, certname), error_ok=True)
     utils.interpret_puppet_exitcodes(rc, output)
 
+    try:
         ssh_client.ssh("reboot")
+    except Exception as e:
+        # Some init system kill the connection too fast after reboot.
+        # Deal with it by ignoring ssh errors when rebooting.
+        if e.rc == -1:
+            pass
+        else:
+            raise
 
 
 def build_server(

launch/sshclient.py

@@ -23,6 +23,12 @@ import sys
 import paramiko
 
 
+class SSHException(Exception):
+    def __init__(self, message, rc):
+        super(SSHException, self).__init__(message)
+        self.rc = rc
+
+
 class SSHClient(object):
     def __init__(self, ip, username, password=None, pkey=None):
         client = paramiko.SSHClient()
@@ -41,7 +47,7 @@ class SSHClient(object):
         ret = stdout.channel.recv_exit_status()
         print stderr.read()
         if (not error_ok) and ret:
-            raise Exception("Unable to %s" % command)
+            raise SSHException("Unable to %s" % command, ret)
         return ret, output
 
     def scp(self, source, dest):