Don't retry forever when a provider is stopped

When we stop a provider, it does not accept any new tasks.  In that
case, we should raise a specific exception so that loops that retry
for a very long time can exit early since the "error" will never
recover.

Change-Id: I7387849ffdb96c487bae4c31ca7cbdd07b8afd8a

nodepool/provider_manager.py

@@ -29,7 +29,7 @@ import keystoneclient.v2_0.client as ksclient
 import time
 
 import fakeprovider
-from task_manager import Task, TaskManager
+from task_manager import Task, TaskManager, ManagerStoppedException
 
 
 SERVER_LIST_AGE = 5   # How long to keep a cached copy of the server list
@@ -423,6 +423,8 @@ class ProviderManager(TaskManager):
                     resource = self.getImage(resource_id)
             except NotFound:
                 continue
+            except ManagerStoppedException:
+                raise
             except Exception:
                 self.log.exception('Unable to list %ss while waiting for '
                                    '%s will retry' % (resource_type,
@@ -474,6 +476,8 @@ class ProviderManager(TaskManager):
                                      (server_id, self.provider.name)):
             try:
                 newip = self.getFloatingIP(ip['id'])
+            except ManagerStoppedException:
+                raise
             except Exception:
                 self.log.exception('Unable to get IP details for server %s, '
                                    'will retry' % (server_id))

nodepool/task_manager.py

@@ -23,6 +23,10 @@ import logging
 import time
 
 
+class ManagerStoppedException(Exception):
+    pass
+
+
 class Task(object):
     def __init__(self, **kw):
         self._wait_event = threading.Event()
@@ -93,6 +97,7 @@ class TaskManager(threading.Thread):
 
     def submitTask(self, task):
         if not self._running:
-            raise Exception("Manager %s is no longer running" % self.name)
+            raise ManagerStoppedException(
+                "Manager %s is no longer running" % self.name)
         self.queue.put(task)
         return task.wait()