Merge "Allow LoopingCall to continue on exception in callee"

oslo_service/loopingcall.py

@@ -46,6 +46,21 @@ class LoopingCallDone(Exception):
         self.retvalue = retvalue
 
 
+def safe_wrapper(f, kind):
+    def func(*args, **kwargs):
+        try:
+            return f(*args, **kwargs)
+        except LoopingCallDone:
+            raise  # let the outer handler process this
+        except Exception:
+            LOG.error(_LE('%(kind)s %(func_name)r failed'),
+                      {'kind': kind, 'func_name': f},
+                      exc_info=True)
+            return 0
+
+    return func
+
+
 class LoopingCallBase(object):
     _KIND = _("Unknown looping call")
 
@@ -70,26 +85,28 @@ class LoopingCallBase(object):
         self._thread = None
         self._running = False
 
-    def _start(self, idle_for, initial_delay=None):
+    def _start(self, idle_for, initial_delay=None, stop_on_exception=True):
         if self._thread is not None:
             raise RuntimeError(self._RUN_ONLY_ONE_MESSAGE)
         self._running = True
         self.done = event.Event()
         self._thread = greenthread.spawn(
             self._run_loop, self._KIND, self.done, idle_for,
-            initial_delay=initial_delay)
+            initial_delay=initial_delay, stop_on_exception=stop_on_exception)
         self._thread.link(self._on_done)
         return self.done
 
     def _run_loop(self, kind, event, idle_for_func,
-                  initial_delay=None):
+                  initial_delay=None, stop_on_exception=True):
+        func = self.f if stop_on_exception else safe_wrapper(self.f, kind)
+
         if initial_delay:
             greenthread.sleep(initial_delay)
         try:
             watch = timeutils.StopWatch()
             while self._running:
                 watch.restart()
-                result = self.f(*self.args, **self.kw)
+                result = func(*self.args, **self.kw)
                 watch.stop()
                 if not self._running:
                     break
@@ -123,7 +140,7 @@ class FixedIntervalLoopingCall(LoopingCallBase):
 
     _KIND = _('Fixed interval looping call')
 
-    def start(self, interval, initial_delay=None):
+    def start(self, interval, initial_delay=None, stop_on_exception=True):
         def _idle_for(result, elapsed):
             delay = elapsed - interval
             if delay > 0:
@@ -131,7 +148,8 @@ class FixedIntervalLoopingCall(LoopingCallBase):
                                 'interval by %(delay).2f sec'),
                             {'func_name': self.f, 'delay': delay})
             return -delay if delay < 0 else 0
-        return self._start(_idle_for, initial_delay=initial_delay)
+        return self._start(_idle_for, initial_delay=initial_delay,
+                           stop_on_exception=stop_on_exception)
 
 
 class DynamicLoopingCall(LoopingCallBase):
@@ -146,13 +164,15 @@ class DynamicLoopingCall(LoopingCallBase):
 
     _KIND = _('Dynamic interval looping call')
 
-    def start(self, initial_delay=None, periodic_interval_max=None):
+    def start(self, initial_delay=None, periodic_interval_max=None,
+              stop_on_exception=True):
         def _idle_for(suggested_delay, elapsed):
             delay = suggested_delay
             if periodic_interval_max is not None:
                 delay = min(delay, periodic_interval_max)
             return delay
-        return self._start(_idle_for, initial_delay=initial_delay)
+        return self._start(_idle_for, initial_delay=initial_delay,
+                           stop_on_exception=stop_on_exception)
 
 
 class RetryDecorator(object):

oslo_service/tests/test_loopingcall.py

@@ -46,6 +46,20 @@ class LoopingCallTestCase(test_base.BaseTestCase):
         timer = loopingcall.FixedIntervalLoopingCall(_raise_it)
         self.assertRaises(RuntimeError, timer.start(interval=0.5).wait)
 
+    def _raise_and_then_done(self):
+        if self.num_runs == 0:
+            raise loopingcall.LoopingCallDone(False)
+        else:
+            self.num_runs = self.num_runs - 1
+            raise RuntimeError()
+
+    def test_do_not_stop_on_exception(self):
+        self.num_runs = 2
+
+        timer = loopingcall.FixedIntervalLoopingCall(self._raise_and_then_done)
+        res = timer.start(interval=0.5, stop_on_exception=False).wait()
+        self.assertFalse(res)
+
     def _wait_for_zero(self):
         """Called at an interval until num_runs == 0."""
         if self.num_runs == 0:
@@ -150,6 +164,19 @@ class DynamicLoopingCallTestCase(test_base.BaseTestCase):
         timer = loopingcall.DynamicLoopingCall(_raise_it)
         self.assertRaises(RuntimeError, timer.start().wait)
 
+    def _raise_and_then_done(self):
+        if self.num_runs == 0:
+            raise loopingcall.LoopingCallDone(False)
+        else:
+            self.num_runs = self.num_runs - 1
+            raise RuntimeError()
+
+    def test_do_not_stop_on_exception(self):
+        self.num_runs = 2
+
+        timer = loopingcall.DynamicLoopingCall(self._raise_and_then_done)
+        timer.start(stop_on_exception=False).wait()
+
     def _wait_for_zero(self):
         """Called at an interval until num_runs == 0."""
         if self.num_runs == 0: