diff --git a/aodh/evaluator/composite.py b/aodh/evaluator/composite.py index 93dc437af..bf1de7750 100644 --- a/aodh/evaluator/composite.py +++ b/aodh/evaluator/composite.py @@ -17,6 +17,7 @@ import six import stevedore from aodh import evaluator +from aodh.evaluator import threshold from aodh.i18n import _ LOG = log.getLogger(__name__) @@ -43,8 +44,13 @@ class RuleTarget(object): if not self.evaluated: LOG.debug('Evaluating %(type)s rule: %(rule)s', {'type': self.type, 'rule': self.rule}) - self.state, self.trending_state, self.statistics, __ = \ - self.rule_evaluator.evaluate_rule(self.rule) + try: + self.state, self.trending_state, self.statistics, __, __ = \ + self.rule_evaluator.evaluate_rule(self.rule) + except threshold.InsufficientDataError as e: + self.state = evaluator.UNKNOWN + self.trending_state = None + self.statistics = e.statistics self.evaluated = True diff --git a/aodh/evaluator/gnocchi.py b/aodh/evaluator/gnocchi.py index 1f7b6ee42..39bcbf094 100644 --- a/aodh/evaluator/gnocchi.py +++ b/aodh/evaluator/gnocchi.py @@ -48,6 +48,9 @@ class GnocchiBase(threshold.ThresholdEvaluator): LOG.debug('sanitize stats %s', statistics) statistics = [stats[VALUE] for stats in statistics if stats[GRANULARITY] == rule['granularity']] + if not statistics: + raise threshold.InsufficientDataError( + "No datapoint for granularity %s" % rule['granularity'], []) statistics = statistics[-rule['evaluation_periods']:] LOG.debug('pruned statistics to %d', len(statistics)) return statistics @@ -61,13 +64,28 @@ class GnocchiResourceThresholdEvaluator(GnocchiBase): start=start, stop=end, resource_id=rule['resource_id'], aggregation=rule['aggregation_method']) + except exceptions.MetricNotFound: + raise threshold.InsufficientDataError( + 'metric %s for resource %s does not exists' % + (rule['metric'], rule['resource_id']), []) + except exceptions.ResourceNotFound: + raise threshold.InsufficientDataError( + 'resource %s does not exists' % rule['resource_id'], []) except exceptions.NotFound: - LOG.debug('metric %s or resource %s does not exists', - rule['metric'], rule['resource_id']) - return [] + # TODO(sileht): gnocchiclient should raise a explicit + # exception for AggregationNotFound, this API endpoint + # can only raise 3 different 404, so we are safe to + # assume this is an AggregationNotFound for now. + raise threshold.InsufficientDataError( + 'aggregation %s does not exist for ' + 'metric %s of resource %s' % (rule['aggregation_method'], + rule['metric'], + rule['resource_id']), + []) except Exception as e: - LOG.warning('alarm stats retrieval failed: %s', e) - return [] + msg = 'alarm statistics retrieval failed: %s' % e + LOG.warning(msg) + raise threshold.InsufficientDataError(msg, []) class GnocchiAggregationMetricsThresholdEvaluator(GnocchiBase): @@ -86,12 +104,23 @@ class GnocchiAggregationMetricsThresholdEvaluator(GnocchiBase): start=start, stop=end, aggregation=rule['aggregation_method'], needed_overlap=0) + except exceptions.MetricNotFound: + raise threshold.InsufficientDataError( + 'At least of metrics in %s does not exist' % + rule['metrics'], []) except exceptions.NotFound: - LOG.debug('metrics %s does not exists', rule['metrics']) - return [] + # TODO(sileht): gnocchiclient should raise a explicit + # exception for AggregationNotFound, this API endpoint + # can only raise 3 different 404, so we are safe to + # assume this is an AggregationNotFound for now. + raise threshold.InsufficientDataError( + 'aggregation %s does not exist for at least one ' + 'metrics in %s' % (rule['aggregation_method'], + rule['metrics']), []) except Exception as e: - LOG.warning('alarm stats retrieval failed: %s', e) - return [] + msg = 'alarm statistics retrieval failed: %s' % e + LOG.warning(msg) + raise threshold.InsufficientDataError(msg, []) class GnocchiAggregationResourcesThresholdEvaluator(GnocchiBase): @@ -113,9 +142,18 @@ class GnocchiAggregationResourcesThresholdEvaluator(GnocchiBase): aggregation=rule['aggregation_method'], needed_overlap=0, ) + except exceptions.MetricNotFound: + raise threshold.InsufficientDataError( + 'metric %s does not exists' % rule['metric'], []) except exceptions.NotFound: - LOG.debug('metric %s does not exists', rule['metric']) - return [] + # TODO(sileht): gnocchiclient should raise a explicit + # exception for AggregationNotFound, this API endpoint + # can only raise 3 different 404, so we are safe to + # assume this is an AggregationNotFound for now. + raise threshold.InsufficientDataError( + 'aggregation %s does not exist for at least one ' + 'metric of the query' % rule['aggregation_method'], []) except Exception as e: - LOG.warning('alarm stats retrieval failed: %s', e) - return [] + msg = 'alarm statistics retrieval failed: %s' % e + LOG.warning(msg) + raise threshold.InsufficientDataError(msg, []) diff --git a/aodh/evaluator/threshold.py b/aodh/evaluator/threshold.py index 714b95f60..884714b1c 100644 --- a/aodh/evaluator/threshold.py +++ b/aodh/evaluator/threshold.py @@ -49,6 +49,13 @@ OPTS = [ ] +class InsufficientDataError(Exception): + def __init__(self, reason, statistics): + self.reason = reason + self.statistics = statistics + super(InsufficientDataError, self).__init__(reason) + + class ThresholdEvaluator(evaluator.Evaluator): # the sliding evaluation window is extended to allow @@ -172,7 +179,9 @@ class ThresholdEvaluator(evaluator.Evaluator): statistics = self._sanitize(alarm_rule, statistics) sufficient = len(statistics) >= alarm_rule['evaluation_periods'] if not sufficient: - return evaluator.UNKNOWN, None, statistics, len(statistics) + raise InsufficientDataError( + '%d datapoints are unknown' % alarm_rule['evaluation_periods'], + statistics) def _compare(value): op = COMPARATORS[alarm_rule['comparison_operator']] @@ -188,13 +197,13 @@ class ThresholdEvaluator(evaluator.Evaluator): if unequivocal: state = evaluator.ALARM if distilled else evaluator.OK - return state, None, statistics, number_outside + return state, None, statistics, number_outside, None else: trending_state = evaluator.ALARM if compared[-1] else evaluator.OK - return None, trending_state, statistics, number_outside + return None, trending_state, statistics, number_outside, None def _transition_alarm(self, alarm, state, trending_state, statistics, - outside_count): + outside_count, unknown_reason): unknown = alarm.state == evaluator.UNKNOWN continuous = alarm.repeat_actions @@ -213,13 +222,11 @@ class ThresholdEvaluator(evaluator.Evaluator): 'actual': len(statistics)}) # Reason is not same as log message because we want to keep # consistent since thirdparty software may depend on old format. - reason = _('%d datapoints are unknown') % alarm.rule[ - 'evaluation_periods'] last = None if not statistics else statistics[-1] reason_data = self._reason_data('unknown', alarm.rule['evaluation_periods'], last) - self._refresh(alarm, state, reason, reason_data) + self._refresh(alarm, state, unknown_reason, reason_data) elif state and (alarm.state != state or continuous): reason, reason_data = self._reason(alarm, statistics, state, @@ -232,7 +239,9 @@ class ThresholdEvaluator(evaluator.Evaluator): 'within its time constraint.', alarm.alarm_id) return - state, trending_state, statistics, outside_count = self.evaluate_rule( - alarm.rule) - self._transition_alarm(alarm, state, trending_state, statistics, - outside_count) + try: + evaluation = self.evaluate_rule(alarm.rule) + except InsufficientDataError as e: + evaluation = (evaluator.UNKNOWN, None, e.statistics, 0, + e.reason) + self._transition_alarm(alarm, *evaluation) diff --git a/aodh/tests/unit/evaluator/test_gnocchi.py b/aodh/tests/unit/evaluator/test_gnocchi.py index c1d78409a..a3122065e 100644 --- a/aodh/tests/unit/evaluator/test_gnocchi.py +++ b/aodh/tests/unit/evaluator/test_gnocchi.py @@ -150,8 +150,8 @@ class TestGnocchiEvaluatorBase(base.TestEvaluatorBase): expected = [mock.call( alarm, 'ok', - ('%d datapoints are unknown' - % alarm.rule['evaluation_periods']), + ('No datapoint for granularity %s' + % alarm.rule['granularity']), self._reason_data('unknown', alarm.rule['evaluation_periods'], None))