Implement the combination evaluator
The implementation of the combination evaluator. 'and' and 'or' operations are allowed. Implements blueprint alarming-logical-combination Change-Id: Ie6b7dedc8aa5debb250c83e1c6db05c0d66eea1b
This commit is contained in:
Mehdi Abaakouk
parent
d30bf2fa2e
commit
985f482709
@ -28,6 +28,10 @@ from ceilometer.openstack.common.gettextutils import _
|
||||
|
||||
LOG = log.getLogger(__name__)
|
||||
|
||||
UNKNOWN = 'insufficient data'
|
||||
OK = 'ok'
|
||||
ALARM = 'alarm'
|
||||
|
||||
|
||||
class Evaluator(object):
|
||||
"""Base class for alarm rule evaluator plugins."""
|
||||
|
||||
103
ceilometer/alarm/evaluator/combination.py
Normal file
103
ceilometer/alarm/evaluator/combination.py
Normal file
@ -0,0 +1,103 @@
|
||||
# -*- encoding: utf-8 -*-
|
||||
#
|
||||
# Copyright © 2013 eNovance <licensing@enovance.com>
|
||||
#
|
||||
# Authors: Mehdi Abaakouk <mehdi.abaakouk@enovance.com>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
from ceilometer.alarm import evaluator
|
||||
from ceilometer.alarm.evaluator import OK, ALARM, UNKNOWN
|
||||
from ceilometer.openstack.common import log
|
||||
from ceilometer.openstack.common.gettextutils import _
|
||||
|
||||
LOG = log.getLogger(__name__)
|
||||
|
||||
COMPARATORS = {'and': all, 'or': any}
|
||||
|
||||
|
||||
class CombinationEvaluator(evaluator.Evaluator):
|
||||
|
||||
def _get_alarm_state(self, alarm_id):
|
||||
try:
|
||||
alarm = self._client.alarm.get(alarm_id)
|
||||
except Exception:
|
||||
LOG.exception(_('alarm retrieval failed'))
|
||||
return None
|
||||
return alarm.state
|
||||
|
||||
def _sufficient_states(self, alarm, states):
|
||||
"""Ensure there is sufficient data for evaluation,
|
||||
transitioning to unknown otherwise.
|
||||
"""
|
||||
missing_states = len(alarm.rule['alarm_ids']) - len(states)
|
||||
sufficient = missing_states == 0
|
||||
if not sufficient and alarm.state != UNKNOWN:
|
||||
reason = _('%(missing_states)d alarms in %(alarm_ids)s'
|
||||
' are in unknown state') % \
|
||||
{'missing_states': missing_states,
|
||||
'alarm_ids': ",".join(alarm.rule['alarm_ids'])}
|
||||
self._refresh(alarm, UNKNOWN, reason)
|
||||
return sufficient
|
||||
|
||||
@staticmethod
|
||||
def _reason(alarm, state):
|
||||
"""Fabricate reason string."""
|
||||
transition = alarm.state != state
|
||||
if alarm.rule['operator'] == 'or':
|
||||
if transition:
|
||||
return (_('Transition to %(state)s due at least to one alarm'
|
||||
' in %(alarm_ids)s in state %(state)s') %
|
||||
{'state': state,
|
||||
'alarm_ids': ",".join(alarm.rule['alarm_ids'])})
|
||||
return (_('Remaining as %(state)s due at least to one alarm in'
|
||||
' %(alarm_ids)s in state %(state)s') %
|
||||
{'state': state,
|
||||
'alarm_ids': ",".join(alarm.rule['alarm_ids'])})
|
||||
elif alarm.rule['operator'] == 'and':
|
||||
if transition:
|
||||
return (_('Transition to %(state)s due to all alarms'
|
||||
' (%(alarm_ids)s) in state %(state)s') %
|
||||
{'state': state,
|
||||
'alarm_ids': ",".join(alarm.rule['alarm_ids'])})
|
||||
return (_('Remaining as %(state)s due to all alarms'
|
||||
' (%(alarm_ids)s) in state %(state)s') %
|
||||
{'state': state,
|
||||
'alarm_ids': ",".join(alarm.rule['alarm_ids'])})
|
||||
|
||||
def _transition(self, alarm, underlying_states):
|
||||
"""Transition alarm state if necessary.
|
||||
"""
|
||||
op = alarm.rule['operator']
|
||||
if COMPARATORS[op](s == ALARM for s in underlying_states):
|
||||
state = ALARM
|
||||
else:
|
||||
state = OK
|
||||
|
||||
continuous = alarm.repeat_actions
|
||||
reason = self._reason(alarm, state)
|
||||
if alarm.state != state or continuous:
|
||||
self._refresh(alarm, state, reason)
|
||||
|
||||
def evaluate(self, alarm):
|
||||
states = []
|
||||
for _id in alarm.rule['alarm_ids']:
|
||||
state = self._get_alarm_state(_id)
|
||||
#note(sileht): alarm can be evaluated only with
|
||||
#stable state of other alarm
|
||||
if state and state != UNKNOWN:
|
||||
states.append(state)
|
||||
|
||||
if self._sufficient_states(alarm, states):
|
||||
self._transition(alarm, states)
|
||||
@ -21,6 +21,7 @@ import datetime
|
||||
import operator
|
||||
|
||||
from ceilometer.alarm import evaluator
|
||||
from ceilometer.alarm.evaluator import OK, ALARM, UNKNOWN
|
||||
from ceilometer.openstack.common import log
|
||||
from ceilometer.openstack.common import timeutils
|
||||
from ceilometer.openstack.common.gettextutils import _
|
||||
@ -36,10 +37,6 @@ COMPARATORS = {
|
||||
'ne': operator.ne,
|
||||
}
|
||||
|
||||
UNKNOWN = 'insufficient data'
|
||||
OK = 'ok'
|
||||
ALARM = 'alarm'
|
||||
|
||||
|
||||
class ThresholdEvaluator(evaluator.Evaluator):
|
||||
|
||||
|
||||
281
tests/alarm/evaluator/test_combination.py
Normal file
281
tests/alarm/evaluator/test_combination.py
Normal file
@ -0,0 +1,281 @@
|
||||
# -*- encoding: utf-8 -*-
|
||||
#
|
||||
# Copyright © 2013 eNovance <licensing@enovance.com>
|
||||
#
|
||||
# Authors: Mehdi Abaakouk <mehdi.abaakouk@enovance.com>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""Tests for ceilometer/alarm/threshold_evaluation.py
|
||||
"""
|
||||
import mock
|
||||
import uuid
|
||||
|
||||
from ceilometer.alarm.evaluator import combination
|
||||
from ceilometer.storage import models
|
||||
from ceilometerclient import exc
|
||||
from ceilometerclient.v2 import alarms
|
||||
|
||||
from tests.alarm.evaluator import base
|
||||
|
||||
|
||||
class TestEvaluate(base.TestEvaluatorBase):
|
||||
EVALUATOR = combination.CombinationEvaluator
|
||||
|
||||
def prepare_alarms(self):
|
||||
self.alarms = [
|
||||
models.Alarm(name='or-alarm',
|
||||
description='the or alarm',
|
||||
type='combination',
|
||||
enabled=True,
|
||||
user_id='foobar',
|
||||
project_id='snafu',
|
||||
alarm_id=str(uuid.uuid4()),
|
||||
state='insufficient data',
|
||||
state_timestamp=None,
|
||||
timestamp=None,
|
||||
insufficient_data_actions=[],
|
||||
ok_actions=[],
|
||||
alarm_actions=[],
|
||||
repeat_actions=False,
|
||||
rule=dict(
|
||||
alarm_ids=[
|
||||
'9cfc3e51-2ff1-4b1d-ac01-c1bd4c6d0d1e',
|
||||
'1d441595-d069-4e05-95ab-8693ba6a8302'],
|
||||
operator='or',
|
||||
)),
|
||||
models.Alarm(name='and-alarm',
|
||||
description='the and alarm',
|
||||
type='combination',
|
||||
enabled=True,
|
||||
user_id='foobar',
|
||||
project_id='snafu',
|
||||
alarm_id=str(uuid.uuid4()),
|
||||
state='insufficient data',
|
||||
state_timestamp=None,
|
||||
timestamp=None,
|
||||
insufficient_data_actions=[],
|
||||
ok_actions=[],
|
||||
alarm_actions=[],
|
||||
repeat_actions=False,
|
||||
rule=dict(
|
||||
alarm_ids=[
|
||||
'b82734f4-9d06-48f3-8a86-fa59a0c99dc8',
|
||||
'15a700e5-2fe8-4b3d-8c55-9e92831f6a2b'],
|
||||
operator='and',
|
||||
))
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _get_alarm(state):
|
||||
return alarms.Alarm(None, {'state': state})
|
||||
|
||||
def _combination_transition_reason(self, state):
|
||||
return ['Transition to %(state)s due at least to one alarm in'
|
||||
' 9cfc3e51-2ff1-4b1d-ac01-c1bd4c6d0d1e,'
|
||||
'1d441595-d069-4e05-95ab-8693ba6a8302'
|
||||
' in state %(state)s' % {'state': state},
|
||||
'Transition to %(state)s due to all alarms'
|
||||
' (b82734f4-9d06-48f3-8a86-fa59a0c99dc8,'
|
||||
'15a700e5-2fe8-4b3d-8c55-9e92831f6a2b)'
|
||||
' in state %(state)s' % {'state': state}]
|
||||
|
||||
def _combination_remaining_reason(self, state):
|
||||
return ['Remaining as %(state)s due at least to one alarm in'
|
||||
' 9cfc3e51-2ff1-4b1d-ac01-c1bd4c6d0d1e,'
|
||||
'1d441595-d069-4e05-95ab-8693ba6a8302'
|
||||
' in state %(state)s' % {'state': state},
|
||||
'Remaining as %(state)s due to all alarms'
|
||||
' (b82734f4-9d06-48f3-8a86-fa59a0c99dc8,'
|
||||
'15a700e5-2fe8-4b3d-8c55-9e92831f6a2b)'
|
||||
' in state %(state)s' % {'state': state}]
|
||||
|
||||
def test_retry_transient_api_failure(self):
|
||||
with mock.patch('ceilometerclient.client.get_client',
|
||||
return_value=self.api_client):
|
||||
broken = exc.CommunicationError(message='broken')
|
||||
self.api_client.alarm.get.side_effect = [
|
||||
broken,
|
||||
broken,
|
||||
broken,
|
||||
broken,
|
||||
self._get_alarm('ok'),
|
||||
self._get_alarm('ok'),
|
||||
self._get_alarm('ok'),
|
||||
self._get_alarm('ok'),
|
||||
]
|
||||
self._evaluate_all_alarms()
|
||||
self._assert_all_alarms('insufficient data')
|
||||
self._evaluate_all_alarms()
|
||||
self._assert_all_alarms('ok')
|
||||
|
||||
def test_simple_insufficient(self):
|
||||
self._set_all_alarms('ok')
|
||||
with mock.patch('ceilometerclient.client.get_client',
|
||||
return_value=self.api_client):
|
||||
broken = exc.CommunicationError(message='broken')
|
||||
self.api_client.alarm.get.side_effect = broken
|
||||
self._evaluate_all_alarms()
|
||||
self._assert_all_alarms('insufficient data')
|
||||
expected = [mock.call(alarm.alarm_id, state='insufficient data')
|
||||
for alarm in self.alarms]
|
||||
update_calls = self.api_client.alarms.update.call_args_list
|
||||
self.assertEqual(update_calls, expected)
|
||||
expected = [mock.call(alarm,
|
||||
'ok',
|
||||
('%d alarms in %s are in unknown state' %
|
||||
(2, ",".join(alarm.rule['alarm_ids']))))
|
||||
for alarm in self.alarms]
|
||||
self.assertEqual(self.notifier.notify.call_args_list, expected)
|
||||
|
||||
def test_to_ok_with_all_ok(self):
|
||||
self._set_all_alarms('insufficient data')
|
||||
with mock.patch('ceilometerclient.client.get_client',
|
||||
return_value=self.api_client):
|
||||
self.api_client.alarm.get.side_effect = [
|
||||
self._get_alarm('ok'),
|
||||
self._get_alarm('ok'),
|
||||
self._get_alarm('ok'),
|
||||
self._get_alarm('ok'),
|
||||
]
|
||||
self._evaluate_all_alarms()
|
||||
expected = [mock.call(alarm.alarm_id, state='ok')
|
||||
for alarm in self.alarms]
|
||||
update_calls = self.api_client.alarms.update.call_args_list
|
||||
self.assertEqual(update_calls, expected)
|
||||
reasons = self._combination_transition_reason('ok')
|
||||
expected = [mock.call(alarm, 'insufficient data', reason)
|
||||
for alarm, reason in zip(self.alarms, reasons)]
|
||||
self.assertEqual(self.notifier.notify.call_args_list, expected)
|
||||
|
||||
def test_to_ok_with_one_alarm(self):
|
||||
self._set_all_alarms('alarm')
|
||||
with mock.patch('ceilometerclient.client.get_client',
|
||||
return_value=self.api_client):
|
||||
self.api_client.alarm.get.side_effect = [
|
||||
self._get_alarm('ok'),
|
||||
self._get_alarm('ok'),
|
||||
self._get_alarm('alarm'),
|
||||
self._get_alarm('ok'),
|
||||
]
|
||||
self._evaluate_all_alarms()
|
||||
expected = [mock.call(alarm.alarm_id, state='ok')
|
||||
for alarm in self.alarms]
|
||||
update_calls = self.api_client.alarms.update.call_args_list
|
||||
self.assertEqual(update_calls, expected)
|
||||
reasons = self._combination_transition_reason('ok')
|
||||
expected = [mock.call(alarm, 'alarm', reason)
|
||||
for alarm, reason in zip(self.alarms, reasons)]
|
||||
self.assertEqual(self.notifier.notify.call_args_list, expected)
|
||||
|
||||
def test_to_alarm_with_all_alarm(self):
|
||||
self._set_all_alarms('ok')
|
||||
with mock.patch('ceilometerclient.client.get_client',
|
||||
return_value=self.api_client):
|
||||
self.api_client.alarm.get.side_effect = [
|
||||
self._get_alarm('alarm'),
|
||||
self._get_alarm('alarm'),
|
||||
self._get_alarm('alarm'),
|
||||
self._get_alarm('alarm'),
|
||||
]
|
||||
self._evaluate_all_alarms()
|
||||
expected = [mock.call(alarm.alarm_id, state='alarm')
|
||||
for alarm in self.alarms]
|
||||
update_calls = self.api_client.alarms.update.call_args_list
|
||||
self.assertEqual(update_calls, expected)
|
||||
reasons = self._combination_transition_reason('alarm')
|
||||
expected = [mock.call(alarm, 'ok', reason)
|
||||
for alarm, reason in zip(self.alarms, reasons)]
|
||||
self.assertEqual(self.notifier.notify.call_args_list, expected)
|
||||
|
||||
def test_to_alarm_with_one_ok(self):
|
||||
self._set_all_alarms('ok')
|
||||
with mock.patch('ceilometerclient.client.get_client',
|
||||
return_value=self.api_client):
|
||||
self.api_client.alarm.get.side_effect = [
|
||||
self._get_alarm('ok'),
|
||||
self._get_alarm('alarm'),
|
||||
self._get_alarm('alarm'),
|
||||
self._get_alarm('alarm'),
|
||||
]
|
||||
self._evaluate_all_alarms()
|
||||
expected = [mock.call(alarm.alarm_id, state='alarm')
|
||||
for alarm in self.alarms]
|
||||
update_calls = self.api_client.alarms.update.call_args_list
|
||||
self.assertEqual(update_calls, expected)
|
||||
reasons = self._combination_transition_reason('alarm')
|
||||
expected = [mock.call(alarm, 'ok', reason)
|
||||
for alarm, reason in zip(self.alarms, reasons)]
|
||||
self.assertEqual(self.notifier.notify.call_args_list, expected)
|
||||
|
||||
def test_to_unknown(self):
|
||||
self._set_all_alarms('ok')
|
||||
with mock.patch('ceilometerclient.client.get_client',
|
||||
return_value=self.api_client):
|
||||
broken = exc.CommunicationError(message='broken')
|
||||
self.api_client.alarm.get.side_effect = [
|
||||
broken,
|
||||
self._get_alarm('ok'),
|
||||
self._get_alarm('insufficient data'),
|
||||
self._get_alarm('ok'),
|
||||
]
|
||||
self._evaluate_all_alarms()
|
||||
expected = [mock.call(alarm.alarm_id, state='insufficient data')
|
||||
for alarm in self.alarms]
|
||||
update_calls = self.api_client.alarms.update.call_args_list
|
||||
self.assertEqual(update_calls, expected)
|
||||
reasons = ['1 alarms in'
|
||||
' 9cfc3e51-2ff1-4b1d-ac01-c1bd4c6d0d1e,'
|
||||
'1d441595-d069-4e05-95ab-8693ba6a8302'
|
||||
' are in unknown state',
|
||||
'1 alarms in'
|
||||
' b82734f4-9d06-48f3-8a86-fa59a0c99dc8,'
|
||||
'15a700e5-2fe8-4b3d-8c55-9e92831f6a2b'
|
||||
' are in unknown state']
|
||||
expected = [mock.call(alarm, 'ok', reason)
|
||||
for alarm, reason in zip(self.alarms, reasons)]
|
||||
self.assertEqual(self.notifier.notify.call_args_list, expected)
|
||||
|
||||
def test_no_state_change(self):
|
||||
self._set_all_alarms('ok')
|
||||
with mock.patch('ceilometerclient.client.get_client',
|
||||
return_value=self.api_client):
|
||||
self.api_client.alarm.get.side_effect = [
|
||||
self._get_alarm('ok'),
|
||||
self._get_alarm('ok'),
|
||||
self._get_alarm('ok'),
|
||||
self._get_alarm('ok'),
|
||||
]
|
||||
self._evaluate_all_alarms()
|
||||
update_calls = self.api_client.alarms.update.call_args_list
|
||||
self.assertEqual(update_calls, [])
|
||||
self.assertEqual(self.notifier.notify.call_args_list, [])
|
||||
|
||||
def test_no_state_change_and_repeat_actions(self):
|
||||
self.alarms[0].repeat_actions = True
|
||||
self.alarms[1].repeat_actions = True
|
||||
self._set_all_alarms('ok')
|
||||
with mock.patch('ceilometerclient.client.get_client',
|
||||
return_value=self.api_client):
|
||||
self.api_client.alarm.get.side_effect = [
|
||||
self._get_alarm('ok'),
|
||||
self._get_alarm('ok'),
|
||||
self._get_alarm('ok'),
|
||||
self._get_alarm('ok'),
|
||||
]
|
||||
self._evaluate_all_alarms()
|
||||
update_calls = self.api_client.alarms.update.call_args_list
|
||||
self.assertEqual(update_calls, [])
|
||||
reasons = self._combination_remaining_reason('ok')
|
||||
expected = [mock.call(alarm, 'ok', reason)
|
||||
for alarm, reason in zip(self.alarms, reasons)]
|
||||
self.assertEqual(self.notifier.notify.call_args_list, expected)
|
||||
Loading…
x
Reference in New Issue
Block a user