Implement liveness check for static nodes

The node liveness will be checked during request handling to make sure
no dead nodes are assigned. Unreachable nodes will be deregistered.

Change-Id: I0c1c9adbaec07419026239746603eba9234b4c8e

nodepool/driver/static/handler.py

@@ -44,3 +44,6 @@ class StaticNodeRequestHandler(NodeRequestHandler):
     def launchesComplete(self):
         # We don't wait on a launch since we never actually launch.
         return True
+
+    def checkReusableNode(self, node):
+        return self.manager.checkNodeLiveness(node)

nodepool/driver/static/provider.py

@@ -78,6 +78,27 @@ class StaticNodeProvider(Provider):
             nodes.append(node)
         return nodes
 
+    def checkNodeLiveness(self, node):
+        static_node = self.poolNodes().get(node.hostname)
+        if static_node is None:
+            return False
+
+        try:
+            nodeutils.nodescan(static_node["name"],
+                               port=static_node["connection-port"],
+                               timeout=static_node["timeout"],
+                               gather_hostkeys=False)
+            return True
+        except Exception:
+            self.log.exception("Failed to connect to node %s:",
+                               static_node["name"])
+        try:
+            self.deregisterNode(count=1, node_name=static_node["name"])
+        except Exception:
+            self.log.exception("Couldn't deregister static node:")
+
+        return False
+
     def getRegisteredNodeHostnames(self):
         '''
         Get hostnames for all registered static nodes.

nodepool/tests/test_driver_static.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import logging
+import mock
 import os
 
 from nodepool import config as nodepool_config
@@ -278,6 +279,32 @@ class TestDriverStatic(tests.DBTestCase):
         self.assertEqual(len(new_nodes), 1)
         self.assertEqual(nodes[0].hostname, new_nodes[0].hostname)
 
+    def test_liveness_check(self):
+        '''
+        Test liveness check during request handling.
+        '''
+        configfile = self.setup_config('static-basic.yaml')
+        pool = self.useNodepool(configfile, watermark_sleep=1)
+        pool.start()
+        nodes = self.waitForNodes('fake-label')
+        self.assertEqual(len(nodes), 1)
+
+        req = zk.NodeRequest()
+        req.state = zk.REQUESTED
+        req.node_types.append('fake-label')
+
+        with mock.patch("nodepool.nodeutils.nodescan") as nodescan_mock:
+            nodescan_mock.side_effect = OSError
+            self.zk.storeNodeRequest(req)
+            self.waitForNodeDeletion(nodes[0])
+
+        self.log.debug("Waiting for request %s", req.id)
+        req = self.waitForNodeRequest(req)
+
+        self.assertEqual(req.state, zk.FULFILLED)
+        self.assertEqual(len(req.nodes), 1)
+        self.assertNotEqual(req.nodes[0], nodes[0].id)
+
     def test_missing_static_node(self):
         """Test that a missing static node is added"""
         configfile = self.setup_config('static-2-nodes.yaml')