Make event-alarm evaluator caching alarms
This patch enables event-alarm evaluator to cache alarms per project when it get those alarms from the DB. So the evaluator won't access the DB while the cache exists and does not expired. Cached alarms will be updated when the alarm fired for consistency. 'event_alarm_cache_ttl' is added to config. DocImpact Change-Id: I4e6aa14947cf060c44e82c5f43dc02be586eb855 Implements: blueprint event-alarm-evaluator
This commit is contained in:
Ryota MIBU
parent
c9155d4160
commit
217723ce6c
@ -17,7 +17,9 @@ import copy
|
||||
import fnmatch
|
||||
import operator
|
||||
|
||||
from oslo_config import cfg
|
||||
from oslo_log import log
|
||||
from oslo_utils import timeutils
|
||||
|
||||
from aodh import evaluator
|
||||
from aodh.i18n import _, _LE
|
||||
@ -33,9 +35,20 @@ COMPARATORS = {
|
||||
'ne': operator.ne,
|
||||
}
|
||||
|
||||
OPTS = [
|
||||
cfg.IntOpt('event_alarm_cache_ttl',
|
||||
default=60,
|
||||
help='TTL of event alarm caches, in seconds. '
|
||||
'Set to 0 to disable caching.'),
|
||||
]
|
||||
|
||||
|
||||
class EventAlarmEvaluator(evaluator.Evaluator):
|
||||
|
||||
def __init__(self, conf, notifier):
|
||||
super(EventAlarmEvaluator, self).__init__(conf, notifier)
|
||||
self.caches = {}
|
||||
|
||||
def evaluate_events(self, events):
|
||||
"""Evaluate the events by referring related alarms."""
|
||||
|
||||
@ -52,10 +65,7 @@ class EventAlarmEvaluator(evaluator.Evaluator):
|
||||
continue
|
||||
|
||||
project = self._get_project(event)
|
||||
# TODO(r-mibu): cache alarms to reduce DB accesses
|
||||
alarms = self._storage_conn.get_alarms(enabled=True,
|
||||
alarm_type='event',
|
||||
project=project)
|
||||
alarms = self._get_project_alarms(project)
|
||||
LOG.debug('Found %(num)d alarms related to the event '
|
||||
'(message_id=%(id)s)',
|
||||
{'num': len(alarms), 'id': event['message_id']})
|
||||
@ -98,6 +108,26 @@ class EventAlarmEvaluator(evaluator.Evaluator):
|
||||
return trait[2]
|
||||
return ''
|
||||
|
||||
def _get_project_alarms(self, project):
|
||||
if self.conf.event_alarm_cache_ttl and project in self.caches:
|
||||
if timeutils.is_older_than(self.caches[project]['updated'],
|
||||
self.conf.event_alarm_cache_ttl):
|
||||
del self.caches[project]
|
||||
else:
|
||||
return self.caches[project]['alarms']
|
||||
|
||||
alarms = self._storage_conn.get_alarms(enabled=True,
|
||||
alarm_type='event',
|
||||
project=project)
|
||||
|
||||
if self.conf.event_alarm_cache_ttl:
|
||||
self.caches[project] = {
|
||||
'alarms': alarms,
|
||||
'updated': timeutils.utcnow()
|
||||
}
|
||||
|
||||
return alarms
|
||||
|
||||
@staticmethod
|
||||
def _sanitize(event):
|
||||
"""Change traits format to dict."""
|
||||
@ -164,6 +194,18 @@ class EventAlarmEvaluator(evaluator.Evaluator):
|
||||
reason_data = {'type': 'event', 'event': event}
|
||||
self._refresh(alarm, state, reason, reason_data)
|
||||
|
||||
def _refresh(self, alarm, state, reason, reason_data):
|
||||
super(EventAlarmEvaluator, self)._refresh(alarm, state,
|
||||
reason, reason_data)
|
||||
|
||||
project = alarm.project_id
|
||||
if self.conf.event_alarm_cache_ttl and project in self.caches:
|
||||
for index, a in enumerate(self.caches[project]['alarms']):
|
||||
if a.alarm_id == alarm.alarm_id:
|
||||
alarm.state = state
|
||||
self.caches[project]['alarms'][index] = alarm
|
||||
break
|
||||
|
||||
# NOTE(r-mibu): This method won't be used, but we have to define here in
|
||||
# order to overwrite the abstract method in the super class.
|
||||
# TODO(r-mibu): Change the base (common) class design for evaluators.
|
||||
|
||||
@ -19,6 +19,7 @@ import aodh.api
|
||||
import aodh.api.controllers.v2.alarms
|
||||
import aodh.coordination
|
||||
import aodh.evaluator
|
||||
import aodh.evaluator.event
|
||||
import aodh.evaluator.gnocchi
|
||||
import aodh.event
|
||||
import aodh.notifier.rest
|
||||
@ -32,6 +33,7 @@ def list_opts():
|
||||
('DEFAULT',
|
||||
itertools.chain(
|
||||
aodh.evaluator.OPTS,
|
||||
aodh.evaluator.event.OPTS,
|
||||
aodh.evaluator.gnocchi.OPTS,
|
||||
aodh.event.OPTS,
|
||||
aodh.notifier.rest.OPTS,
|
||||
|
||||
@ -12,9 +12,12 @@
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import datetime
|
||||
import uuid
|
||||
|
||||
import mock
|
||||
from oslo_utils import timeutils
|
||||
|
||||
from aodh import evaluator
|
||||
from aodh.evaluator import event as event_evaluator
|
||||
@ -55,7 +58,7 @@ class TestEventAlarmEvaluate(base.TestEvaluatorBase):
|
||||
'traits': kwargs.get('traits', [])}
|
||||
|
||||
def _do_test_event_alarm(self, alarms, events,
|
||||
expect_project_in_query=None,
|
||||
expect_db_queries=None,
|
||||
expect_alarm_states=None,
|
||||
expect_alarm_updates=None,
|
||||
expect_notifications=None):
|
||||
@ -63,10 +66,11 @@ class TestEventAlarmEvaluate(base.TestEvaluatorBase):
|
||||
|
||||
self.evaluator.evaluate_events(events)
|
||||
|
||||
if expect_project_in_query is not None:
|
||||
self.assertEqual([mock.call(enabled=True,
|
||||
if expect_db_queries is not None:
|
||||
expected = [mock.call(enabled=True,
|
||||
alarm_type='event',
|
||||
project=expect_project_in_query)],
|
||||
project=p) for p in expect_db_queries]
|
||||
self.assertEqual(expected,
|
||||
self.storage_conn.get_alarms.call_args_list)
|
||||
if expect_alarm_states is not None:
|
||||
for expected, alarm in zip(expect_alarm_states, alarms):
|
||||
@ -91,7 +95,7 @@ class TestEventAlarmEvaluate(base.TestEvaluatorBase):
|
||||
alarm = self._alarm(project='project1')
|
||||
event = self._event(traits=[['project_id', 1, 'project1']])
|
||||
self._do_test_event_alarm([alarm], [event],
|
||||
expect_project_in_query='project1',
|
||||
expect_db_queries=['project1'],
|
||||
expect_alarm_states=[evaluator.ALARM],
|
||||
expect_alarm_updates=[alarm],
|
||||
expect_notifications=[dict(alarm=alarm,
|
||||
@ -101,7 +105,7 @@ class TestEventAlarmEvaluate(base.TestEvaluatorBase):
|
||||
alarm = self._alarm(project='project1')
|
||||
event = self._event(traits=[['tenant_id', 1, 'project1']])
|
||||
self._do_test_event_alarm([alarm], [event],
|
||||
expect_project_in_query='project1',
|
||||
expect_db_queries=['project1'],
|
||||
expect_alarm_states=[evaluator.ALARM],
|
||||
expect_alarm_updates=[alarm],
|
||||
expect_notifications=[dict(alarm=alarm,
|
||||
@ -111,7 +115,7 @@ class TestEventAlarmEvaluate(base.TestEvaluatorBase):
|
||||
alarm = self._alarm(project='')
|
||||
event = self._event()
|
||||
self._do_test_event_alarm([alarm], [event],
|
||||
expect_project_in_query='',
|
||||
expect_db_queries=[''],
|
||||
expect_alarm_states=[evaluator.ALARM],
|
||||
expect_alarm_updates=[alarm],
|
||||
expect_notifications=[dict(alarm=alarm,
|
||||
@ -214,3 +218,61 @@ class TestEventAlarmEvaluate(base.TestEvaluatorBase):
|
||||
expect_alarm_states=[evaluator.UNKNOWN],
|
||||
expect_alarm_updates=[],
|
||||
expect_notifications=[])
|
||||
|
||||
def test_event_alarm_cache_hit(self):
|
||||
alarm = self._alarm(project='project2', event_type='none')
|
||||
events = [
|
||||
self._event(traits=[['project_id', 1, 'project2']]),
|
||||
self._event(traits=[['project_id', 1, 'project2']]),
|
||||
]
|
||||
self._do_test_event_alarm([alarm], events,
|
||||
expect_db_queries=['project2'])
|
||||
|
||||
def test_event_alarm_cache_updated_after_fired(self):
|
||||
alarm = self._alarm(project='project2', event_type='type1',
|
||||
repeat=False)
|
||||
events = [
|
||||
self._event(event_type='type1',
|
||||
traits=[['project_id', 1, 'project2']]),
|
||||
self._event(event_type='type1',
|
||||
traits=[['project_id', 1, 'project2']]),
|
||||
]
|
||||
self._do_test_event_alarm([alarm], events,
|
||||
expect_db_queries=['project2'],
|
||||
expect_alarm_states=[evaluator.ALARM],
|
||||
expect_alarm_updates=[alarm],
|
||||
expect_notifications=[dict(alarm=alarm,
|
||||
event=events[0])])
|
||||
|
||||
def test_event_alarm_caching_disabled(self):
|
||||
alarm = self._alarm(project='project2', event_type='none')
|
||||
events = [
|
||||
self._event(traits=[['project_id', 1, 'project2']]),
|
||||
self._event(traits=[['project_id', 1, 'project2']]),
|
||||
]
|
||||
self.evaluator.conf.event_alarm_cache_ttl = 0
|
||||
self._do_test_event_alarm([alarm], events,
|
||||
expect_db_queries=['project2', 'project2'])
|
||||
|
||||
@mock.patch.object(timeutils, 'utcnow')
|
||||
def test_event_alarm_cache_expired(self, mock_utcnow):
|
||||
alarm = self._alarm(project='project2', event_type='none')
|
||||
events = [
|
||||
self._event(traits=[['project_id', 1, 'project2']]),
|
||||
self._event(traits=[['project_id', 1, 'project2']]),
|
||||
]
|
||||
mock_utcnow.side_effect = [
|
||||
datetime.datetime(2015, 1, 1, 0, 0, 0),
|
||||
datetime.datetime(2015, 1, 1, 1, 0, 0),
|
||||
datetime.datetime(2015, 1, 1, 1, 1, 0),
|
||||
]
|
||||
self._do_test_event_alarm([alarm], events,
|
||||
expect_db_queries=['project2', 'project2'])
|
||||
|
||||
def test_event_alarm_cache_miss(self):
|
||||
events = [
|
||||
self._event(traits=[['project_id', 1, 'project2']]),
|
||||
self._event(traits=[['project_id', 1, 'project3']]),
|
||||
]
|
||||
self._do_test_event_alarm([], events,
|
||||
expect_db_queries=['project2', 'project3'])
|
||||
|
||||
Loading…
Reference in New Issue
Block a user