diff --git a/oslo_service/loopingcall.py b/oslo_service/loopingcall.py index d8eb651a..e4312477 100644 --- a/oslo_service/loopingcall.py +++ b/oslo_service/loopingcall.py @@ -20,6 +20,7 @@ import sys from eventlet import event from eventlet import greenthread +from oslo_utils import excutils from oslo_utils import timeutils from oslo_service._i18n import _LE, _LW, _ @@ -151,3 +152,82 @@ class DynamicLoopingCall(LoopingCallBase): delay = min(delay, periodic_interval_max) return delay return self._start(_idle_for, initial_delay=initial_delay) + + +class RetryDecorator(object): + """Decorator for retrying a function upon suggested exceptions. + + The decorated function is retried for the given number of times, and the + sleep time between the retries is incremented until max sleep time is + reached. If the max retry count is set to -1, then the decorated function + is invoked indefinitely until an exception is thrown, and the caught + exception is not in the list of suggested exceptions. + """ + + def __init__(self, max_retry_count=-1, inc_sleep_time=10, + max_sleep_time=60, exceptions=()): + """Configure the retry object using the input params. + + :param max_retry_count: maximum number of times the given function must + be retried when one of the input 'exceptions' + is caught. When set to -1, it will be retried + indefinitely until an exception is thrown + and the caught exception is not in param + exceptions. + :param inc_sleep_time: incremental time in seconds for sleep time + between retries + :param max_sleep_time: max sleep time in seconds beyond which the sleep + time will not be incremented using param + inc_sleep_time. On reaching this threshold, + max_sleep_time will be used as the sleep time. + :param exceptions: suggested exceptions for which the function must be + retried + """ + self._max_retry_count = max_retry_count + self._inc_sleep_time = inc_sleep_time + self._max_sleep_time = max_sleep_time + self._exceptions = exceptions + self._retry_count = 0 + self._sleep_time = 0 + + def __call__(self, f): + + def _func(*args, **kwargs): + func_name = f.__name__ + result = None + try: + if self._retry_count: + LOG.debug("Invoking %(func_name)s; retry count is " + "%(retry_count)d.", + {'func_name': func_name, + 'retry_count': self._retry_count}) + result = f(*args, **kwargs) + except self._exceptions: + with excutils.save_and_reraise_exception() as ctxt: + LOG.warn(_LW("Exception which is in the suggested list of " + "exceptions occurred while invoking function:" + " %s."), + func_name, + exc_info=True) + if (self._max_retry_count != -1 and + self._retry_count >= self._max_retry_count): + LOG.error(_LE("Cannot retry upon suggested exception " + "since retry count (%(retry_count)d) " + "reached max retry count " + "(%(max_retry_count)d)."), + {'retry_count': self._retry_count, + 'max_retry_count': self._max_retry_count}) + else: + ctxt.reraise = False + self._retry_count += 1 + self._sleep_time += self._inc_sleep_time + return self._sleep_time + raise LoopingCallDone(result) + + def func(*args, **kwargs): + loop = DynamicLoopingCall(_func, *args, **kwargs) + evt = loop.start(periodic_interval_max=self._max_sleep_time) + LOG.debug("Waiting for function %s to return.", f.__name__) + return evt.wait() + + return func diff --git a/oslo_service/tests/test_loopingcall.py b/oslo_service/tests/test_loopingcall.py index c528adb1..cd7692c2 100644 --- a/oslo_service/tests/test_loopingcall.py +++ b/oslo_service/tests/test_loopingcall.py @@ -182,3 +182,76 @@ class DynamicLoopingCallTestCase(test_base.BaseTestCase): timer.start(initial_delay=3).wait() sleep_mock.assert_has_calls([mock.call(3), mock.call(1)]) + + +class AnException(Exception): + pass + + +class UnknownException(Exception): + pass + + +class RetryDecoratorTest(test_base.BaseTestCase): + """Tests for retry decorator class.""" + + def test_retry(self): + result = "RESULT" + + @loopingcall.RetryDecorator() + def func(*args, **kwargs): + return result + + self.assertEqual(result, func()) + + def func2(*args, **kwargs): + return result + + retry = loopingcall.RetryDecorator() + self.assertEqual(result, retry(func2)()) + self.assertTrue(retry._retry_count == 0) + + def test_retry_with_expected_exceptions(self): + result = "RESULT" + responses = [AnException(None), + AnException(None), + result] + + def func(*args, **kwargs): + response = responses.pop(0) + if isinstance(response, Exception): + raise response + return response + + sleep_time_incr = 0.01 + retry_count = 2 + retry = loopingcall.RetryDecorator(10, sleep_time_incr, 10, + (AnException,)) + self.assertEqual(result, retry(func)()) + self.assertTrue(retry._retry_count == retry_count) + self.assertEqual(retry_count * sleep_time_incr, retry._sleep_time) + + def test_retry_with_max_retries(self): + responses = [AnException(None), + AnException(None), + AnException(None)] + + def func(*args, **kwargs): + response = responses.pop(0) + if isinstance(response, Exception): + raise response + return response + + retry = loopingcall.RetryDecorator(2, 0, 0, + (AnException,)) + self.assertRaises(AnException, retry(func)) + self.assertTrue(retry._retry_count == 2) + + def test_retry_with_unexpected_exception(self): + + def func(*args, **kwargs): + raise UnknownException(None) + + retry = loopingcall.RetryDecorator() + self.assertRaises(UnknownException, retry(func)) + self.assertTrue(retry._retry_count == 0)