From 217723ce6c3af63469661add21c5f86febfa6950 Mon Sep 17 00:00:00 2001 From: Ryota MIBU Date: Sat, 22 Aug 2015 21:51:24 +0900 Subject: [PATCH] Make event-alarm evaluator caching alarms This patch enables event-alarm evaluator to cache alarms per project when it get those alarms from the DB. So the evaluator won't access the DB while the cache exists and does not expired. Cached alarms will be updated when the alarm fired for consistency. 'event_alarm_cache_ttl' is added to config. DocImpact Change-Id: I4e6aa14947cf060c44e82c5f43dc02be586eb855 Implements: blueprint event-alarm-evaluator --- aodh/evaluator/event.py | 50 +++++++++++++++++-- aodh/opts.py | 2 + aodh/tests/evaluator/test_event.py | 78 +++++++++++++++++++++++++++--- 3 files changed, 118 insertions(+), 12 deletions(-) diff --git a/aodh/evaluator/event.py b/aodh/evaluator/event.py index 0ed9cec5d..9a1d8a4ee 100644 --- a/aodh/evaluator/event.py +++ b/aodh/evaluator/event.py @@ -17,7 +17,9 @@ import copy import fnmatch import operator +from oslo_config import cfg from oslo_log import log +from oslo_utils import timeutils from aodh import evaluator from aodh.i18n import _, _LE @@ -33,9 +35,20 @@ COMPARATORS = { 'ne': operator.ne, } +OPTS = [ + cfg.IntOpt('event_alarm_cache_ttl', + default=60, + help='TTL of event alarm caches, in seconds. ' + 'Set to 0 to disable caching.'), +] + class EventAlarmEvaluator(evaluator.Evaluator): + def __init__(self, conf, notifier): + super(EventAlarmEvaluator, self).__init__(conf, notifier) + self.caches = {} + def evaluate_events(self, events): """Evaluate the events by referring related alarms.""" @@ -52,10 +65,7 @@ class EventAlarmEvaluator(evaluator.Evaluator): continue project = self._get_project(event) - # TODO(r-mibu): cache alarms to reduce DB accesses - alarms = self._storage_conn.get_alarms(enabled=True, - alarm_type='event', - project=project) + alarms = self._get_project_alarms(project) LOG.debug('Found %(num)d alarms related to the event ' '(message_id=%(id)s)', {'num': len(alarms), 'id': event['message_id']}) @@ -98,6 +108,26 @@ class EventAlarmEvaluator(evaluator.Evaluator): return trait[2] return '' + def _get_project_alarms(self, project): + if self.conf.event_alarm_cache_ttl and project in self.caches: + if timeutils.is_older_than(self.caches[project]['updated'], + self.conf.event_alarm_cache_ttl): + del self.caches[project] + else: + return self.caches[project]['alarms'] + + alarms = self._storage_conn.get_alarms(enabled=True, + alarm_type='event', + project=project) + + if self.conf.event_alarm_cache_ttl: + self.caches[project] = { + 'alarms': alarms, + 'updated': timeutils.utcnow() + } + + return alarms + @staticmethod def _sanitize(event): """Change traits format to dict.""" @@ -164,6 +194,18 @@ class EventAlarmEvaluator(evaluator.Evaluator): reason_data = {'type': 'event', 'event': event} self._refresh(alarm, state, reason, reason_data) + def _refresh(self, alarm, state, reason, reason_data): + super(EventAlarmEvaluator, self)._refresh(alarm, state, + reason, reason_data) + + project = alarm.project_id + if self.conf.event_alarm_cache_ttl and project in self.caches: + for index, a in enumerate(self.caches[project]['alarms']): + if a.alarm_id == alarm.alarm_id: + alarm.state = state + self.caches[project]['alarms'][index] = alarm + break + # NOTE(r-mibu): This method won't be used, but we have to define here in # order to overwrite the abstract method in the super class. # TODO(r-mibu): Change the base (common) class design for evaluators. diff --git a/aodh/opts.py b/aodh/opts.py index fe3a46884..8f7cd7f7f 100644 --- a/aodh/opts.py +++ b/aodh/opts.py @@ -19,6 +19,7 @@ import aodh.api import aodh.api.controllers.v2.alarms import aodh.coordination import aodh.evaluator +import aodh.evaluator.event import aodh.evaluator.gnocchi import aodh.event import aodh.notifier.rest @@ -32,6 +33,7 @@ def list_opts(): ('DEFAULT', itertools.chain( aodh.evaluator.OPTS, + aodh.evaluator.event.OPTS, aodh.evaluator.gnocchi.OPTS, aodh.event.OPTS, aodh.notifier.rest.OPTS, diff --git a/aodh/tests/evaluator/test_event.py b/aodh/tests/evaluator/test_event.py index b14bae8a4..4c320de45 100644 --- a/aodh/tests/evaluator/test_event.py +++ b/aodh/tests/evaluator/test_event.py @@ -12,9 +12,12 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. + +import datetime import uuid import mock +from oslo_utils import timeutils from aodh import evaluator from aodh.evaluator import event as event_evaluator @@ -55,7 +58,7 @@ class TestEventAlarmEvaluate(base.TestEvaluatorBase): 'traits': kwargs.get('traits', [])} def _do_test_event_alarm(self, alarms, events, - expect_project_in_query=None, + expect_db_queries=None, expect_alarm_states=None, expect_alarm_updates=None, expect_notifications=None): @@ -63,10 +66,11 @@ class TestEventAlarmEvaluate(base.TestEvaluatorBase): self.evaluator.evaluate_events(events) - if expect_project_in_query is not None: - self.assertEqual([mock.call(enabled=True, - alarm_type='event', - project=expect_project_in_query)], + if expect_db_queries is not None: + expected = [mock.call(enabled=True, + alarm_type='event', + project=p) for p in expect_db_queries] + self.assertEqual(expected, self.storage_conn.get_alarms.call_args_list) if expect_alarm_states is not None: for expected, alarm in zip(expect_alarm_states, alarms): @@ -91,7 +95,7 @@ class TestEventAlarmEvaluate(base.TestEvaluatorBase): alarm = self._alarm(project='project1') event = self._event(traits=[['project_id', 1, 'project1']]) self._do_test_event_alarm([alarm], [event], - expect_project_in_query='project1', + expect_db_queries=['project1'], expect_alarm_states=[evaluator.ALARM], expect_alarm_updates=[alarm], expect_notifications=[dict(alarm=alarm, @@ -101,7 +105,7 @@ class TestEventAlarmEvaluate(base.TestEvaluatorBase): alarm = self._alarm(project='project1') event = self._event(traits=[['tenant_id', 1, 'project1']]) self._do_test_event_alarm([alarm], [event], - expect_project_in_query='project1', + expect_db_queries=['project1'], expect_alarm_states=[evaluator.ALARM], expect_alarm_updates=[alarm], expect_notifications=[dict(alarm=alarm, @@ -111,7 +115,7 @@ class TestEventAlarmEvaluate(base.TestEvaluatorBase): alarm = self._alarm(project='') event = self._event() self._do_test_event_alarm([alarm], [event], - expect_project_in_query='', + expect_db_queries=[''], expect_alarm_states=[evaluator.ALARM], expect_alarm_updates=[alarm], expect_notifications=[dict(alarm=alarm, @@ -214,3 +218,61 @@ class TestEventAlarmEvaluate(base.TestEvaluatorBase): expect_alarm_states=[evaluator.UNKNOWN], expect_alarm_updates=[], expect_notifications=[]) + + def test_event_alarm_cache_hit(self): + alarm = self._alarm(project='project2', event_type='none') + events = [ + self._event(traits=[['project_id', 1, 'project2']]), + self._event(traits=[['project_id', 1, 'project2']]), + ] + self._do_test_event_alarm([alarm], events, + expect_db_queries=['project2']) + + def test_event_alarm_cache_updated_after_fired(self): + alarm = self._alarm(project='project2', event_type='type1', + repeat=False) + events = [ + self._event(event_type='type1', + traits=[['project_id', 1, 'project2']]), + self._event(event_type='type1', + traits=[['project_id', 1, 'project2']]), + ] + self._do_test_event_alarm([alarm], events, + expect_db_queries=['project2'], + expect_alarm_states=[evaluator.ALARM], + expect_alarm_updates=[alarm], + expect_notifications=[dict(alarm=alarm, + event=events[0])]) + + def test_event_alarm_caching_disabled(self): + alarm = self._alarm(project='project2', event_type='none') + events = [ + self._event(traits=[['project_id', 1, 'project2']]), + self._event(traits=[['project_id', 1, 'project2']]), + ] + self.evaluator.conf.event_alarm_cache_ttl = 0 + self._do_test_event_alarm([alarm], events, + expect_db_queries=['project2', 'project2']) + + @mock.patch.object(timeutils, 'utcnow') + def test_event_alarm_cache_expired(self, mock_utcnow): + alarm = self._alarm(project='project2', event_type='none') + events = [ + self._event(traits=[['project_id', 1, 'project2']]), + self._event(traits=[['project_id', 1, 'project2']]), + ] + mock_utcnow.side_effect = [ + datetime.datetime(2015, 1, 1, 0, 0, 0), + datetime.datetime(2015, 1, 1, 1, 0, 0), + datetime.datetime(2015, 1, 1, 1, 1, 0), + ] + self._do_test_event_alarm([alarm], events, + expect_db_queries=['project2', 'project2']) + + def test_event_alarm_cache_miss(self): + events = [ + self._event(traits=[['project_id', 1, 'project2']]), + self._event(traits=[['project_id', 1, 'project3']]), + ] + self._do_test_event_alarm([], events, + expect_db_queries=['project2', 'project3'])