Make event-alarm evaluator caching alarms

This patch enables event-alarm evaluator to cache alarms per project
when it get those alarms from the DB. So the evaluator won't access the
DB while the cache exists and does not expired. Cached alarms will be
updated when the alarm fired for consistency.

'event_alarm_cache_ttl' is added to config.

DocImpact

Change-Id: I4e6aa14947cf060c44e82c5f43dc02be586eb855
Implements: blueprint event-alarm-evaluator

aodh/evaluator/event.py

@@ -17,7 +17,9 @@ import copy
 import fnmatch
 import operator
 
+from oslo_config import cfg
 from oslo_log import log
+from oslo_utils import timeutils
 
 from aodh import evaluator
 from aodh.i18n import _, _LE
@@ -33,9 +35,20 @@ COMPARATORS = {
     'ne': operator.ne,
 }
 
+OPTS = [
+    cfg.IntOpt('event_alarm_cache_ttl',
+               default=60,
+               help='TTL of event alarm caches, in seconds. '
+                    'Set to 0 to disable caching.'),
+]
+
 
 class EventAlarmEvaluator(evaluator.Evaluator):
 
+    def __init__(self, conf, notifier):
+        super(EventAlarmEvaluator, self).__init__(conf, notifier)
+        self.caches = {}
+
     def evaluate_events(self, events):
         """Evaluate the events by referring related alarms."""
 
@@ -52,10 +65,7 @@ class EventAlarmEvaluator(evaluator.Evaluator):
                 continue
 
             project = self._get_project(event)
-            # TODO(r-mibu): cache alarms to reduce DB accesses
+            alarms = self._get_project_alarms(project)
-            alarms = self._storage_conn.get_alarms(enabled=True,
-                                                   alarm_type='event',
-                                                   project=project)
             LOG.debug('Found %(num)d alarms related to the event '
                       '(message_id=%(id)s)',
                       {'num': len(alarms), 'id': event['message_id']})
@@ -98,6 +108,26 @@ class EventAlarmEvaluator(evaluator.Evaluator):
                 return trait[2]
         return ''
 
+    def _get_project_alarms(self, project):
+        if self.conf.event_alarm_cache_ttl and project in self.caches:
+            if timeutils.is_older_than(self.caches[project]['updated'],
+                                       self.conf.event_alarm_cache_ttl):
+                del self.caches[project]
+            else:
+                return self.caches[project]['alarms']
+
+        alarms = self._storage_conn.get_alarms(enabled=True,
+                                               alarm_type='event',
+                                               project=project)
+
+        if self.conf.event_alarm_cache_ttl:
+            self.caches[project] = {
+                'alarms': alarms,
+                'updated': timeutils.utcnow()
+            }
+
+        return alarms
+
     @staticmethod
     def _sanitize(event):
         """Change traits format to dict."""
@@ -164,6 +194,18 @@ class EventAlarmEvaluator(evaluator.Evaluator):
         reason_data = {'type': 'event', 'event': event}
         self._refresh(alarm, state, reason, reason_data)
 
+    def _refresh(self, alarm, state, reason, reason_data):
+        super(EventAlarmEvaluator, self)._refresh(alarm, state,
+                                                  reason, reason_data)
+
+        project = alarm.project_id
+        if self.conf.event_alarm_cache_ttl and project in self.caches:
+            for index, a in enumerate(self.caches[project]['alarms']):
+                if a.alarm_id == alarm.alarm_id:
+                    alarm.state = state
+                    self.caches[project]['alarms'][index] = alarm
+                    break
+
     # NOTE(r-mibu): This method won't be used, but we have to define here in
     # order to overwrite the abstract method in the super class.
     # TODO(r-mibu): Change the base (common) class design for evaluators.

aodh/opts.py

@@ -19,6 +19,7 @@ import aodh.api
 import aodh.api.controllers.v2.alarms
 import aodh.coordination
 import aodh.evaluator
+import aodh.evaluator.event
 import aodh.evaluator.gnocchi
 import aodh.event
 import aodh.notifier.rest
@@ -32,6 +33,7 @@ def list_opts():
         ('DEFAULT',
          itertools.chain(
              aodh.evaluator.OPTS,
+             aodh.evaluator.event.OPTS,
              aodh.evaluator.gnocchi.OPTS,
              aodh.event.OPTS,
              aodh.notifier.rest.OPTS,

aodh/tests/evaluator/test_event.py

@@ -12,9 +12,12 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
+
+import datetime
 import uuid
 
 import mock
+from oslo_utils import timeutils
 
 from aodh import evaluator
 from aodh.evaluator import event as event_evaluator
@@ -55,7 +58,7 @@ class TestEventAlarmEvaluate(base.TestEvaluatorBase):
                 'traits': kwargs.get('traits', [])}
 
     def _do_test_event_alarm(self, alarms, events,
-                             expect_project_in_query=None,
+                             expect_db_queries=None,
                              expect_alarm_states=None,
                              expect_alarm_updates=None,
                              expect_notifications=None):
@@ -63,10 +66,11 @@ class TestEventAlarmEvaluate(base.TestEvaluatorBase):
 
         self.evaluator.evaluate_events(events)
 
-        if expect_project_in_query is not None:
+        if expect_db_queries is not None:
-            self.assertEqual([mock.call(enabled=True,
+            expected = [mock.call(enabled=True,
-                                        alarm_type='event',
+                                  alarm_type='event',
-                                        project=expect_project_in_query)],
+                                  project=p) for p in expect_db_queries]
+            self.assertEqual(expected,
                              self.storage_conn.get_alarms.call_args_list)
         if expect_alarm_states is not None:
             for expected, alarm in zip(expect_alarm_states, alarms):
@@ -91,7 +95,7 @@ class TestEventAlarmEvaluate(base.TestEvaluatorBase):
         alarm = self._alarm(project='project1')
         event = self._event(traits=[['project_id', 1, 'project1']])
         self._do_test_event_alarm([alarm], [event],
-                                  expect_project_in_query='project1',
+                                  expect_db_queries=['project1'],
                                   expect_alarm_states=[evaluator.ALARM],
                                   expect_alarm_updates=[alarm],
                                   expect_notifications=[dict(alarm=alarm,
@@ -101,7 +105,7 @@ class TestEventAlarmEvaluate(base.TestEvaluatorBase):
         alarm = self._alarm(project='project1')
         event = self._event(traits=[['tenant_id', 1, 'project1']])
         self._do_test_event_alarm([alarm], [event],
-                                  expect_project_in_query='project1',
+                                  expect_db_queries=['project1'],
                                   expect_alarm_states=[evaluator.ALARM],
                                   expect_alarm_updates=[alarm],
                                   expect_notifications=[dict(alarm=alarm,
@@ -111,7 +115,7 @@ class TestEventAlarmEvaluate(base.TestEvaluatorBase):
         alarm = self._alarm(project='')
         event = self._event()
         self._do_test_event_alarm([alarm], [event],
-                                  expect_project_in_query='',
+                                  expect_db_queries=[''],
                                   expect_alarm_states=[evaluator.ALARM],
                                   expect_alarm_updates=[alarm],
                                   expect_notifications=[dict(alarm=alarm,
@@ -214,3 +218,61 @@ class TestEventAlarmEvaluate(base.TestEvaluatorBase):
                                   expect_alarm_states=[evaluator.UNKNOWN],
                                   expect_alarm_updates=[],
                                   expect_notifications=[])
+
+    def test_event_alarm_cache_hit(self):
+        alarm = self._alarm(project='project2', event_type='none')
+        events = [
+            self._event(traits=[['project_id', 1, 'project2']]),
+            self._event(traits=[['project_id', 1, 'project2']]),
+        ]
+        self._do_test_event_alarm([alarm], events,
+                                  expect_db_queries=['project2'])
+
+    def test_event_alarm_cache_updated_after_fired(self):
+        alarm = self._alarm(project='project2', event_type='type1',
+                            repeat=False)
+        events = [
+            self._event(event_type='type1',
+                        traits=[['project_id', 1, 'project2']]),
+            self._event(event_type='type1',
+                        traits=[['project_id', 1, 'project2']]),
+        ]
+        self._do_test_event_alarm([alarm], events,
+                                  expect_db_queries=['project2'],
+                                  expect_alarm_states=[evaluator.ALARM],
+                                  expect_alarm_updates=[alarm],
+                                  expect_notifications=[dict(alarm=alarm,
+                                                             event=events[0])])
+
+    def test_event_alarm_caching_disabled(self):
+        alarm = self._alarm(project='project2', event_type='none')
+        events = [
+            self._event(traits=[['project_id', 1, 'project2']]),
+            self._event(traits=[['project_id', 1, 'project2']]),
+        ]
+        self.evaluator.conf.event_alarm_cache_ttl = 0
+        self._do_test_event_alarm([alarm], events,
+                                  expect_db_queries=['project2', 'project2'])
+
+    @mock.patch.object(timeutils, 'utcnow')
+    def test_event_alarm_cache_expired(self, mock_utcnow):
+        alarm = self._alarm(project='project2', event_type='none')
+        events = [
+            self._event(traits=[['project_id', 1, 'project2']]),
+            self._event(traits=[['project_id', 1, 'project2']]),
+        ]
+        mock_utcnow.side_effect = [
+            datetime.datetime(2015, 1, 1, 0, 0, 0),
+            datetime.datetime(2015, 1, 1, 1, 0, 0),
+            datetime.datetime(2015, 1, 1, 1, 1, 0),
+        ]
+        self._do_test_event_alarm([alarm], events,
+                                  expect_db_queries=['project2', 'project2'])
+
+    def test_event_alarm_cache_miss(self):
+        events = [
+            self._event(traits=[['project_id', 1, 'project2']]),
+            self._event(traits=[['project_id', 1, 'project3']]),
+        ]
+        self._do_test_event_alarm([], events,
+                                  expect_db_queries=['project2', 'project3'])