From 0e9188d1b5f1aee9965a07846f9742e8c091d72d Mon Sep 17 00:00:00 2001
From: David Shrewsbury <shrewsbury.dave@gmail.com>
Date: Mon, 20 Mar 2017 09:30:54 -0400
Subject: [PATCH] Unlock request if it disappears

Found an issue where we were not unlocking the node request if it
disappeared on us. This caused the request lock cleanup to fail b/c
it remained lock.

Also, let's catch cleanup errors individually so that each phase has
a chance to run, independent of errors from other phases.

Also add recursive=True to the request lock delete.

Change-Id: I12c79b7725460eae5a27063523f3fa2e19e6bc59
---
 nodepool/nodepool.py | 18 +++++++++++++++++-
 nodepool/zk.py       |  2 +-
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/nodepool/nodepool.py b/nodepool/nodepool.py
index 932709707..b17ea68b3 100644
--- a/nodepool/nodepool.py
+++ b/nodepool/nodepool.py
@@ -842,6 +842,7 @@ class NodeRequestHandler(object):
                 node.allocated_to = None
                 self.zk.storeNode(node)
             self.unlockNodeSet()
+            self.zk.unlockNodeRequest(self.request)
             return True
 
         if self.launch_manager.failed_nodes:
@@ -1205,12 +1206,27 @@ class CleanupWorker(BaseCleanupWorker):
                 manager.cleanupLeakedFloaters()
 
     def _run(self):
+        '''
+        Catch exceptions individually so that other cleanup routines may
+        have a chance.
+        '''
         try:
             self._cleanupNodeRequestLocks()
+        except Exception:
+            self.log.exception(
+                "Exception in DeletedNodeWorker (node request lock cleanup):")
+
+        try:
             self._cleanupLeakedInstances()
+        except Exception:
+            self.log.exception(
+                "Exception in DeletedNodeWorker (leaked instance cleanup):")
+
+        try:
             self._cleanupLostRequests()
         except Exception:
-            self.log.exception("Exception in DeletedNodeWorker:")
+            self.log.exception(
+                "Exception in DeletedNodeWorker (lost request cleanup):")
 
 
 class DeletedNodeWorker(BaseCleanupWorker):
diff --git a/nodepool/zk.py b/nodepool/zk.py
index fae26b4c7..a12d966bb 100644
--- a/nodepool/zk.py
+++ b/nodepool/zk.py
@@ -1308,7 +1308,7 @@ class ZooKeeper(object):
         '''
         path = self._requestLockPath(lock)
         try:
-            self.client.delete(path)
+            self.client.delete(path, recursive=True)
         except kze.NoNodeError:
             pass