diff --git a/nodepool/nodepool.py b/nodepool/nodepool.py index 932709707..b17ea68b3 100644 --- a/nodepool/nodepool.py +++ b/nodepool/nodepool.py @@ -842,6 +842,7 @@ class NodeRequestHandler(object): node.allocated_to = None self.zk.storeNode(node) self.unlockNodeSet() + self.zk.unlockNodeRequest(self.request) return True if self.launch_manager.failed_nodes: @@ -1205,12 +1206,27 @@ class CleanupWorker(BaseCleanupWorker): manager.cleanupLeakedFloaters() def _run(self): + ''' + Catch exceptions individually so that other cleanup routines may + have a chance. + ''' try: self._cleanupNodeRequestLocks() + except Exception: + self.log.exception( + "Exception in DeletedNodeWorker (node request lock cleanup):") + + try: self._cleanupLeakedInstances() + except Exception: + self.log.exception( + "Exception in DeletedNodeWorker (leaked instance cleanup):") + + try: self._cleanupLostRequests() except Exception: - self.log.exception("Exception in DeletedNodeWorker:") + self.log.exception( + "Exception in DeletedNodeWorker (lost request cleanup):") class DeletedNodeWorker(BaseCleanupWorker): diff --git a/nodepool/zk.py b/nodepool/zk.py index fae26b4c7..a12d966bb 100644 --- a/nodepool/zk.py +++ b/nodepool/zk.py @@ -1308,7 +1308,7 @@ class ZooKeeper(object): ''' path = self._requestLockPath(lock) try: - self.client.delete(path) + self.client.delete(path, recursive=True) except kze.NoNodeError: pass