diff --git a/ceilometer/api/v1/blueprint.py b/ceilometer/api/v1/blueprint.py index 9f457fc67..ec298d5d2 100644 --- a/ceilometer/api/v1/blueprint.py +++ b/ceilometer/api/v1/blueprint.py @@ -593,6 +593,26 @@ def compute_duration_by_resource(resource, meter): ) +def _get_statistics(stats_type, meter=None, resource=None, project=None): + q_ts = _get_query_timestamps(flask.request.args) + + f = storage.EventFilter( + meter=meter, + project=project, + resource=resource, + start=q_ts['query_start'], + end=q_ts['query_end'], + ) + # TODO(sberler): do we want to return an error if the resource + # does not exist? + results = list(flask.request.storage_conn.get_meter_statistics(f)) + value = None + if results: + value = results[0][stats_type] # there should only be one! + + return flask.jsonify(volume=value) + + @blueprint.route('/resources//meters//volume/max') def compute_max_resource_volume(resource, meter): """Return the max volume for a meter. @@ -606,24 +626,12 @@ def compute_max_resource_volume(resource, meter): :param search_offset: Number of minutes before and after start and end timestamps to query. """ - q_ts = _get_query_timestamps(flask.request.args) - - # Query the database for the max volume - f = storage.EventFilter( + return _get_statistics( + 'max', meter=meter, - project=acl.get_limited_to_project(flask.request.headers), resource=resource, - start=q_ts['query_start'], - end=q_ts['query_end'], + project=acl.get_limited_to_project(flask.request.headers), ) - # TODO(sberler): do we want to return an error if the resource - # does not exist? - results = list(flask.request.storage_conn.get_volume_max(f)) - value = None - if results: - value = results[0].get('value') # there should only be one! - - return flask.jsonify(volume=value) @blueprint.route('/resources//meters//volume/sum') @@ -639,24 +647,12 @@ def compute_resource_volume_sum(resource, meter): :param search_offset: Number of minutes before and after start and end timestamps to query. """ - q_ts = _get_query_timestamps(flask.request.args) - - # Query the database for the max volume - f = storage.EventFilter( + return _get_statistics( + 'sum', meter=meter, - project=acl.get_limited_to_project(flask.request.headers), resource=resource, - start=q_ts['query_start'], - end=q_ts['query_end'], + project=acl.get_limited_to_project(flask.request.headers), ) - # TODO(sberler): do we want to return an error if the resource - # does not exist? - results = list(flask.request.storage_conn.get_volume_sum(f)) - value = None - if results: - value = results[0].get('value') # there should only be one! - - return flask.jsonify(volume=value) @blueprint.route('/projects//meters//volume/max') @@ -673,24 +669,7 @@ def compute_project_volume_max(project, meter): after start and end timestamps to query. """ check_authorized_project(project) - - q_ts = _get_query_timestamps(flask.request.args) - - f = storage.EventFilter(meter=meter, - project=project, - start=q_ts['query_start'], - end=q_ts['query_end'], - ) - # FIXME(sberler): Currently get_volume_max is really always grouping - # by resource_id. We should add a new function in the storage driver - # that does not do this grouping (and potentially rename the existing - # one to get_volume_max_by_resource()) - results = list(flask.request.storage_conn.get_volume_max(f)) - value = None - if results: - value = max(result.get('value') for result in results) - - return flask.jsonify(volume=value) + return _get_statistics('max', project=project, meter=meter) @blueprint.route('/projects//meters//volume/sum') @@ -708,20 +687,8 @@ def compute_project_volume_sum(project, meter): """ check_authorized_project(project) - q_ts = _get_query_timestamps(flask.request.args) - - f = storage.EventFilter(meter=meter, - project=project, - start=q_ts['query_start'], - end=q_ts['query_end'], - ) - # FIXME(sberler): Currently get_volume_max is really always grouping - # by resource_id. We should add a new function in the storage driver - # that does not do this grouping (and potentially rename the existing - # one to get_volume_max_by_resource()) - results = list(flask.request.storage_conn.get_volume_sum(f)) - value = None - if results: - value = sum(result.get('value') for result in results) - - return flask.jsonify(volume=value) + return _get_statistics( + 'sum', + meter=meter, + project=project, + ) diff --git a/ceilometer/storage/base.py b/ceilometer/storage/base.py index af82314d3..313696526 100644 --- a/ceilometer/storage/base.py +++ b/ceilometer/storage/base.py @@ -145,30 +145,6 @@ class Connection(object): :func:`ceilometer.meter.meter_message_from_counter`. """ - @abc.abstractmethod - def get_volume_sum(self, event_filter): - """Return the sum of the volume field for the samples - described by the query parameters. - - The filter must have a meter value set. - - { 'resource_id': UUID string for the resource, - 'value': The sum for the volume. - } - """ - - @abc.abstractmethod - def get_volume_max(self, event_filter): - """Return the maximum of the volume field for the samples - described by the query parameters. - - The filter must have a meter value set. - - { 'resource_id': UUID string for the resource, - 'value': The max for the volume. - } - """ - @abc.abstractmethod def get_event_interval(self, event_filter): """Return the min and max timestamps from samples, diff --git a/ceilometer/storage/impl_hbase.py b/ceilometer/storage/impl_hbase.py index 71b5e5f99..a329288bd 100644 --- a/ceilometer/storage/impl_hbase.py +++ b/ceilometer/storage/impl_hbase.py @@ -478,10 +478,19 @@ class Connection(base.Connection): row_stop=stop) ) - start_time = event_filter.start \ - or timeutils.parse_strtime(meters[-1]['f:timestamp']) - end_time = event_filter.end \ - or timeutils.parse_strtime(meters[0]['f:timestamp']) + if event_filter.start: + start_time = event_filter.start + elif meters: + start_time = timeutils.parse_strtime(meters[-1]['f:timestamp']) + else: + start_time = None + + if event_filter.end: + end_time = event_filter.end + elif meters: + end_time = timeutils.parse_strtime(meters[0]['f:timestamp']) + else: + end_time = None results = [] @@ -519,37 +528,6 @@ class Connection(base.Connection): self._update_meter_stats(results[-1], meter) return list(results) - def get_volume_sum(self, event_filter): - """Return the sum of the volume field for the samples - described by the query parameters. - """ - q, start, stop = make_query_from_filter(event_filter) - LOG.debug("q: %s" % q) - gen = self.meter.scan(filter=q, row_start=start, row_stop=stop) - results = defaultdict(int) - for ignored, meter in gen: - results[meter['f:resource_id']] \ - += int(meter['f:counter_volume']) - - return ({'resource_id': k, 'value': v} - for (k, v) in results.iteritems()) - - def get_volume_max(self, event_filter): - """Return the maximum of the volume field for the samples - described by the query parameters. - """ - - q, start, stop = make_query_from_filter(event_filter) - LOG.debug("q: %s" % q) - gen = self.meter.scan(filter=q, row_start=start, row_stop=stop) - results = defaultdict(int) - for ignored, meter in gen: - results[meter['f:resource_id']] = \ - max(results[meter['f:resource_id']], - int(meter['f:counter_volume'])) - return ({'resource_id': k, 'value': v} - for (k, v) in results.iteritems()) - def get_event_interval(self, event_filter): """Return the min and max timestamps from samples, using the event_filter to limit the samples seen. diff --git a/ceilometer/storage/impl_log.py b/ceilometer/storage/impl_log.py index 09972e228..b46a3df96 100644 --- a/ceilometer/storage/impl_log.py +++ b/ceilometer/storage/impl_log.py @@ -124,16 +124,6 @@ class Connection(base.Connection): """ return [] - def get_volume_sum(self, event_filter): - """Return the sum of the volume field for the samples - described by the query parameters. - """ - - def get_volume_max(self, event_filter): - """Return the maximum of the volume field for the samples - described by the query parameters. - """ - def get_event_interval(self, event_filter): """Return the min and max timestamp for samples matching the event_filter. diff --git a/ceilometer/storage/impl_mongodb.py b/ceilometer/storage/impl_mongodb.py index 9dca70d16..82f7fb317 100644 --- a/ceilometer/storage/impl_mongodb.py +++ b/ceilometer/storage/impl_mongodb.py @@ -132,34 +132,6 @@ class Connection(base.Connection): _mim_instance = None - # JavaScript function for doing map-reduce to get a counter volume - # total. - MAP_COUNTER_VOLUME = bson.code.Code(""" - function() { - emit(this.resource_id, this.counter_volume); - } - """) - - # JavaScript function for doing map-reduce to get a maximum value - # from a range. (from - # http://cookbook.mongodb.org/patterns/finding_max_and_min/) - REDUCE_MAX = bson.code.Code(""" - function (key, values) { - return Math.max.apply(Math, values); - } - """) - - # JavaScript function for doing map-reduce to get a sum. - REDUCE_SUM = bson.code.Code(""" - function (key, values) { - var total = 0; - for (var i = 0; i < values.length; i++) { - total += values[i]; - } - return total; - } - """) - # MAP_TIMESTAMP and REDUCE_MIN_MAX are based on the recipe # http://cookbook.mongodb.org/patterns/finding_max_and_min_values_for_a_key MAP_TIMESTAMP = bson.code.Code(""" @@ -548,32 +520,6 @@ class Connection(base.Connection): return sorted((r['value'] for r in results['results']), key=operator.itemgetter('period_start')) - def get_volume_sum(self, event_filter): - """Return the sum of the volume field for the samples - described by the query parameters. - """ - q = make_query_from_filter(event_filter) - results = self.db.meter.map_reduce(self.MAP_COUNTER_VOLUME, - self.REDUCE_SUM, - {'inline': 1}, - query=q, - ) - return ({'resource_id': r['_id'], 'value': r['value']} - for r in results['results']) - - def get_volume_max(self, event_filter): - """Return the maximum of the volume field for the samples - described by the query parameters. - """ - q = make_query_from_filter(event_filter) - results = self.db.meter.map_reduce(self.MAP_COUNTER_VOLUME, - self.REDUCE_MAX, - {'inline': 1}, - query=q, - ) - return ({'resource_id': r['_id'], 'value': r['value']} - for r in results['results']) - def _fix_interval_min_max(self, a_min, a_max): if hasattr(a_min, 'valueOf') and a_min.valueOf is not None: # NOTE (dhellmann): HACK ALERT diff --git a/ceilometer/storage/impl_sqlalchemy.py b/ceilometer/storage/impl_sqlalchemy.py index ab1589531..73a3c932d 100644 --- a/ceilometer/storage/impl_sqlalchemy.py +++ b/ceilometer/storage/impl_sqlalchemy.py @@ -352,18 +352,6 @@ class Connection(base.Connection): mainq = mainq.join(Meter).group_by(Resource.id) return mainq.filter(Meter.id.in_(subq)) - def get_volume_sum(self, event_filter): - counter_volume_func = func.sum(Meter.counter_volume) - query = self._make_volume_query(event_filter, counter_volume_func) - results = query.all() - return ({'resource_id': x, 'value': y} for x, y in results) - - def get_volume_max(self, event_filter): - counter_volume_func = func.max(Meter.counter_volume) - query = self._make_volume_query(event_filter, counter_volume_func) - results = query.all() - return ({'resource_id': x, 'value': y} for x, y in results) - def get_event_interval(self, event_filter): """Return the min and max timestamps from samples, using the event_filter to limit the samples seen. @@ -398,8 +386,10 @@ class Connection(base.Connection): 'sum': result.sum, 'duration_start': result.tsmin, 'duration_end': result.tsmax, - 'duration': timeutils.delta_seconds(result.tsmin, - result.tsmax), + 'duration': (timeutils.delta_seconds(result.tsmin, + result.tsmax) + if result.tsmin and result.tsmax + else None), 'period': period, 'period_start': period_start, 'period_end': period_end} diff --git a/tests/storage/base.py b/tests/storage/base.py index d765ff239..1fddf4c7f 100644 --- a/tests/storage/base.py +++ b/tests/storage/base.py @@ -342,50 +342,6 @@ class RawEventTest(DBTestBase): assert len(results) == 1 -class SumTest(DBTestBase): - - def test_by_user(self): - f = storage.EventFilter( - user='user-id', - meter='instance', - ) - results = list(self.conn.get_volume_sum(f)) - assert results - counts = dict((r['resource_id'], r['value']) - for r in results) - assert counts['resource-id'] == 1 - assert counts['resource-id-alternate'] == 1 - assert set(counts.keys()) == set(['resource-id', - 'resource-id-alternate']) - - def test_by_project(self): - f = storage.EventFilter( - project='project-id', - meter='instance', - ) - results = list(self.conn.get_volume_sum(f)) - assert results - counts = dict((r['resource_id'], r['value']) - for r in results) - assert counts['resource-id'] == 1 - assert counts['resource-id-alternate'] == 2 - assert set(counts.keys()) == set(['resource-id', - 'resource-id-alternate']) - - def test_one_resource(self): - f = storage.EventFilter( - user='user-id', - meter='instance', - resource='resource-id', - ) - results = list(self.conn.get_volume_sum(f)) - assert results - counts = dict((r['resource_id'], r['value']) - for r in results) - assert counts['resource-id'] == 1 - assert set(counts.keys()) == set(['resource-id']) - - class TestGetEventInterval(DBTestBase): def setUp(self): @@ -481,173 +437,6 @@ class TestGetEventInterval(DBTestBase): assert e is None -class MaxProjectTest(DBTestBase): - - def prepare_data(self): - self.counters = [] - for i in range(3): - c = counter.Counter( - 'volume.size', - 'gauge', - 'GiB', - 5 + i, - 'user-id', - 'project1', - 'resource-id-%s' % i, - timestamp=datetime.datetime(2012, 9, 25, 10 + i, 30 + i), - resource_metadata={'display_name': 'test-volume', - 'tag': 'self.counter', - } - ) - self.counters.append(c) - msg = meter.meter_message_from_counter(c, - cfg.CONF.metering_secret, - 'source1', - ) - self.conn.record_metering_data(msg) - - def test_no_bounds(self): - expected = [{'value': 5.0, 'resource_id': u'resource-id-0'}, - {'value': 6.0, 'resource_id': u'resource-id-1'}, - {'value': 7.0, 'resource_id': u'resource-id-2'}] - - f = storage.EventFilter(project='project1', - meter='volume.size') - - results = list(self.conn.get_volume_max(f)) - assert results == expected - - def test_start_timestamp(self): - expected = [{'value': 6L, 'resource_id': u'resource-id-1'}, - {'value': 7L, 'resource_id': u'resource-id-2'}] - - f = storage.EventFilter(project='project1', - meter='volume.size', - start='2012-09-25T11:30:00') - - results = list(self.conn.get_volume_max(f)) - assert results == expected - - def test_start_timestamp_after(self): - f = storage.EventFilter(project='project1', - meter='volume.size', - start='2012-09-25T12:34:00') - - results = list(self.conn.get_volume_max(f)) - assert results == [] - - def test_end_timestamp(self): - expected = [{'value': 5L, 'resource_id': u'resource-id-0'}] - - f = storage.EventFilter(project='project1', - meter='volume.size', - end='2012-09-25T11:30:00') - - results = list(self.conn.get_volume_max(f)) - assert results == expected - - def test_end_timestamp_before(self): - f = storage.EventFilter(project='project1', - meter='volume.size', - end='2012-09-25T09:54:00') - - results = list(self.conn.get_volume_max(f)) - assert results == [] - - def test_start_end_timestamp(self): - expected = [{'value': 6L, 'resource_id': u'resource-id-1'}] - - f = storage.EventFilter(project='project1', - meter='volume.size', - start='2012-09-25T11:30:00', - end='2012-09-25T11:32:00') - - results = list(self.conn.get_volume_max(f)) - assert results == expected - - -class MaxResourceTest(DBTestBase): - - def prepare_data(self): - self.counters = [] - for i in range(3): - c = counter.Counter( - 'volume.size', - 'gauge', - 'GiB', - 5 + i, - 'user-id', - 'project1', - 'resource-id', - timestamp=datetime.datetime(2012, 9, 25, 10 + i, 30 + i), - resource_metadata={'display_name': 'test-volume', - 'tag': 'self.counter', - } - ) - self.counters.append(c) - msg = meter.meter_message_from_counter(c, - cfg.CONF.metering_secret, - 'source1', - ) - self.conn.record_metering_data(msg) - - def test_no_bounds(self): - expected = [{'value': 7L, 'resource_id': u'resource-id'}] - - f = storage.EventFilter(resource='resource-id', - meter='volume.size') - - results = list(self.conn.get_volume_max(f)) - assert results == expected - - def test_start_timestamp(self): - expected = [{'value': 7L, 'resource_id': u'resource-id'}] - - f = storage.EventFilter(resource='resource-id', - meter='volume.size', - start='2012-09-25T11:30:00') - - results = list(self.conn.get_volume_max(f)) - assert results == expected - - def test_start_timestamp_after(self): - f = storage.EventFilter(resource='resource-id', - meter='volume.size', - start='2012-09-25T12:34:00') - - results = list(self.conn.get_volume_max(f)) - assert results == [] - - def test_end_timestamp(self): - expected = [{'value': 5L, 'resource_id': u'resource-id'}] - - f = storage.EventFilter(resource='resource-id', - meter='volume.size', - end='2012-09-25T11:30:00') - - results = list(self.conn.get_volume_max(f)) - assert results == expected - - def test_end_timestamp_before(self): - f = storage.EventFilter(resource='resource-id', - meter='volume.size', - end='2012-09-25T09:54:00') - - results = list(self.conn.get_volume_max(f)) - assert results == [] - - def test_start_end_timestamp(self): - expected = [{'value': 6L, 'resource_id': u'resource-id'}] - - f = storage.EventFilter(resource='resource-id', - meter='volume.size', - start='2012-09-25T11:30:00', - end='2012-09-25T11:32:00') - - results = list(self.conn.get_volume_max(f)) - assert results == expected - - class StatisticsTest(DBTestBase): def prepare_data(self): diff --git a/tests/storage/test_impl_hbase.py b/tests/storage/test_impl_hbase.py index e0bb22d88..db3b58855 100644 --- a/tests/storage/test_impl_hbase.py +++ b/tests/storage/test_impl_hbase.py @@ -56,18 +56,6 @@ class TestGetEventInterval(base.TestGetEventInterval, pass -class SumTest(base.SumTest, HBaseEngineTestBase): - pass - - -class MaxProjectTest(base.MaxProjectTest, HBaseEngineTestBase): - pass - - -class MaxResourceTest(base.MaxResourceTest, HBaseEngineTestBase): - pass - - class StatisticsTest(base.StatisticsTest, HBaseEngineTestBase): pass diff --git a/tests/storage/test_impl_mongodb.py b/tests/storage/test_impl_mongodb.py index 440f7fd8c..3d163a68e 100644 --- a/tests/storage/test_impl_mongodb.py +++ b/tests/storage/test_impl_mongodb.py @@ -90,13 +90,6 @@ class RawEventTest(base.RawEventTest, MongoDBEngineTestBase): pass -class SumTest(base.SumTest, MongoDBEngineTestBase): - - def setUp(self): - super(SumTest, self).setUp() - require_map_reduce(self.conn) - - class TestGetEventInterval(base.TestGetEventInterval, MongoDBEngineTestBase): def setUp(self): @@ -104,20 +97,6 @@ class TestGetEventInterval(base.TestGetEventInterval, MongoDBEngineTestBase): require_map_reduce(self.conn) -class MaxProjectTest(base.MaxProjectTest, MongoDBEngineTestBase): - - def setUp(self): - super(MaxProjectTest, self).setUp() - require_map_reduce(self.conn) - - -class MaxResourceTest(base.MaxResourceTest, MongoDBEngineTestBase): - - def setUp(self): - super(MaxResourceTest, self).setUp() - require_map_reduce(self.conn) - - class StatisticsTest(base.StatisticsTest, MongoDBEngineTestBase): def setUp(self): diff --git a/tests/storage/test_impl_sqlalchemy.py b/tests/storage/test_impl_sqlalchemy.py index 33ef9acd3..59476a960 100644 --- a/tests/storage/test_impl_sqlalchemy.py +++ b/tests/storage/test_impl_sqlalchemy.py @@ -58,18 +58,6 @@ class TestGetEventInterval(base.TestGetEventInterval, pass -class SumTest(base.SumTest, SQLAlchemyEngineTestBase): - pass - - -class MaxProjectTest(base.MaxProjectTest, SQLAlchemyEngineTestBase): - pass - - -class MaxResourceTest(base.MaxResourceTest, SQLAlchemyEngineTestBase): - pass - - class StatisticsTest(base.StatisticsTest, SQLAlchemyEngineTestBase): pass diff --git a/tools/show_data.py b/tools/show_data.py index c9a835349..8e091f639 100755 --- a/tools/show_data.py +++ b/tools/show_data.py @@ -41,23 +41,20 @@ def show_resources(db, args): for k, v in sorted(resource['metadata'].iteritems()): print ' %-10s : %s' % (k, v) for meter in resource['meter']: + totals = db.get_statistics(storage.EventFilter( + user=u, + meter=meter['counter_name'], + resource=resource['resource_id'], + )) # FIXME(dhellmann): Need a way to tell whether to use # max() or sum() by meter name without hard-coding. if meter['counter_name'] in ['cpu', 'disk']: - totals = db.get_volume_max(storage.EventFilter( - user=u, - meter=meter['counter_name'], - resource=resource['resource_id'], - )) + value = totals[0]['max'] else: - totals = db.get_volume_sum(storage.EventFilter( - user=u, - meter=meter['counter_name'], - resource=resource['resource_id'], - )) + value = totals[0]['sum'] print ' %s (%s): %s' % \ (meter['counter_name'], meter['counter_type'], - totals.next()['value']) + value) def show_total_resources(db, args): @@ -68,18 +65,15 @@ def show_total_resources(db, args): for u in users: print u for meter in ['disk', 'cpu', 'instance']: + stats = db.get_statistics(storage.EventFilter( + user=u, + meter=meter, + )) if meter in ['cpu', 'disk']: - total = db.get_volume_max(storage.EventFilter( - user=u, - meter=meter, - )) + total = stats['max'] else: - total = db.get_volume_sum(storage.EventFilter( - user=u, - meter=meter, - )) - for t in total: - print ' ', meter, t['resource_id'], t['value'] + total = stats['sum'] + print ' ', meter, total def show_raw(db, args):