From aa9daf0629d061d3ecd340cf65da78553d7b37c8 Mon Sep 17 00:00:00 2001
From: "James E. Blair" <jeblair@redhat.com>
Date: Thu, 2 Apr 2020 15:00:39 -0700
Subject: [PATCH] Fix shutdown ordering

The taskmanager driver did not stop the taskmanager thread
correctly.  It is updated to do so.

The launcher shutdown order stopped providers before the cleanup
threads, however, cleanup threads may be in the middle of interacting
with providers, which can cause them to hang.  Instead, stop the
cleanup threads first, then the providers.  Without this, failing
unit tests may not exit correctly, making debugging difficult.

Change-Id: I06256ddca38b815bf61ae3f710e000d666c3ef4b
---
 nodepool/driver/taskmanager.py | 4 ++--
 nodepool/launcher.py           | 6 ++++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/nodepool/driver/taskmanager.py b/nodepool/driver/taskmanager.py
index 7a44bc8d1..a8d8bfcae 100644
--- a/nodepool/driver/taskmanager.py
+++ b/nodepool/driver/taskmanager.py
@@ -146,13 +146,13 @@ class TaskManager:
             while True:
                 task = self.queue.get()
                 if not task:
-                    if not self._running:
-                        break
                     continue
                 self.log.debug("Manager %s running task %s (queue %s)" %
                                (self.name, task.name, self.queue.qsize()))
                 task.run(self)
                 self.queue.task_done()
+                if not self._running:
+                    break
         except Exception:
             self.log.exception("Task manager died")
             raise
diff --git a/nodepool/launcher.py b/nodepool/launcher.py
index 688e14cf4..0976bdf85 100644
--- a/nodepool/launcher.py
+++ b/nodepool/launcher.py
@@ -893,8 +893,6 @@ class NodePool(threading.Thread):
         # completed before we continue the shutdown.
         if self.isAlive():
             self.join()
-        if self.config:
-            provider_manager.ProviderManager.stopProviders(self.config)
 
         if self._cleanup_thread:
             self._cleanup_thread.stop()
@@ -917,6 +915,10 @@ class NodePool(threading.Thread):
             self.log.debug("Waiting for %s" % thd.name)
             thd.join()
 
+        # Stop providers after all the cleanup threads have stopped.
+        if self.config:
+            provider_manager.ProviderManager.stopProviders(self.config)
+
         if self.zk:
             self.zk.disconnect()
         self.log.debug("Finished stopping")