Fix shutdown ordering

The taskmanager driver did not stop the taskmanager thread
correctly.  It is updated to do so.

The launcher shutdown order stopped providers before the cleanup
threads, however, cleanup threads may be in the middle of interacting
with providers, which can cause them to hang.  Instead, stop the
cleanup threads first, then the providers.  Without this, failing
unit tests may not exit correctly, making debugging difficult.

Change-Id: I06256ddca38b815bf61ae3f710e000d666c3ef4b

nodepool/driver/taskmanager.py

@@ -146,13 +146,13 @@ class TaskManager:
             while True:
                 task = self.queue.get()
                 if not task:
-                    if not self._running:
-                        break
                     continue
                 self.log.debug("Manager %s running task %s (queue %s)" %
                                (self.name, task.name, self.queue.qsize()))
                 task.run(self)
                 self.queue.task_done()
+                if not self._running:
+                    break
         except Exception:
             self.log.exception("Task manager died")
             raise

nodepool/launcher.py

@@ -893,8 +893,6 @@ class NodePool(threading.Thread):
         # completed before we continue the shutdown.
         if self.isAlive():
             self.join()
-        if self.config:
-            provider_manager.ProviderManager.stopProviders(self.config)
 
         if self._cleanup_thread:
             self._cleanup_thread.stop()
@@ -917,6 +915,10 @@ class NodePool(threading.Thread):
             self.log.debug("Waiting for %s" % thd.name)
             thd.join()
 
+        # Stop providers after all the cleanup threads have stopped.
+        if self.config:
+            provider_manager.ProviderManager.stopProviders(self.config)
+
         if self.zk:
             self.zk.disconnect()
         self.log.debug("Finished stopping")