From df5049f2e298ed55689baaf132f4122c35f80492 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Wed, 3 Sep 2014 11:13:27 -0700 Subject: [PATCH] Don't retry forever when a provider is stopped When we stop a provider, it does not accept any new tasks. In that case, we should raise a specific exception so that loops that retry for a very long time can exit early since the "error" will never recover. Change-Id: I7387849ffdb96c487bae4c31ca7cbdd07b8afd8a --- nodepool/provider_manager.py | 6 +++++- nodepool/task_manager.py | 7 ++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/nodepool/provider_manager.py b/nodepool/provider_manager.py index 652d9fe7d..d32bd72c8 100644 --- a/nodepool/provider_manager.py +++ b/nodepool/provider_manager.py @@ -29,7 +29,7 @@ import keystoneclient.v2_0.client as ksclient import time import fakeprovider -from task_manager import Task, TaskManager +from task_manager import Task, TaskManager, ManagerStoppedException SERVER_LIST_AGE = 5 # How long to keep a cached copy of the server list @@ -423,6 +423,8 @@ class ProviderManager(TaskManager): resource = self.getImage(resource_id) except NotFound: continue + except ManagerStoppedException: + raise except Exception: self.log.exception('Unable to list %ss while waiting for ' '%s will retry' % (resource_type, @@ -474,6 +476,8 @@ class ProviderManager(TaskManager): (server_id, self.provider.name)): try: newip = self.getFloatingIP(ip['id']) + except ManagerStoppedException: + raise except Exception: self.log.exception('Unable to get IP details for server %s, ' 'will retry' % (server_id)) diff --git a/nodepool/task_manager.py b/nodepool/task_manager.py index 47db55e63..dbae09e40 100644 --- a/nodepool/task_manager.py +++ b/nodepool/task_manager.py @@ -23,6 +23,10 @@ import logging import time +class ManagerStoppedException(Exception): + pass + + class Task(object): def __init__(self, **kw): self._wait_event = threading.Event() @@ -93,6 +97,7 @@ class TaskManager(threading.Thread): def submitTask(self, task): if not self._running: - raise Exception("Manager %s is no longer running" % self.name) + raise ManagerStoppedException( + "Manager %s is no longer running" % self.name) self.queue.put(task) return task.wait()