openstack/aodh
Code Issues Proposed changes

Add date range parameters to resource API

Browse Source 
Allow the caller to query for resources updated within a certain
time range. Changes the storage engine API as well as the web
API.

Also includes a fix for the event date range query, which was
broken when both start and end values were used.

Change-Id: Ic482ea647ad07553a6621ba84ed76cf10d534652
Signed-off-by: Doug Hellmann <doug.hellmann@dreamhost.com>
This commit is contained in:
Doug Hellmann 2012-08-15 20:18:44 -04:00
parent 039f3d893d
commit 7b18eef521
5 changed files with 113 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
ceilometer/api/v1.py
View File

@ -70,6 +70,8 @@
import flask
from ceilometer.openstack.common import log
from ceilometer.openstack.common import timeutils
from ceilometer import storage
@ -82,13 +84,20 @@ blueprint = flask.Blueprint('v1', __name__)
## APIs for working with resources.
def _list_resources(source=None, user=None, project=None):
def _list_resources(source=None, user=None, project=None,
start_timestamp=None, end_timestamp=None):
"""Return a list of resource identifiers.
"""
if start_timestamp:
start_timestamp = timeutils.parse_isotime(start_timestamp)
if end_timestamp:
end_timestamp = timeutils.parse_isotime(end_timestamp)
resources = flask.request.storage_conn.get_resources(
source=source,
user=user,
project=project,
start_timestamp=start_timestamp,
end_timestamp=end_timestamp,
)
return flask.jsonify(resources=list(resources))
@ -99,7 +108,11 @@ def list_resources_by_project(project):
:param project: The ID of the owning project.
"""
return _list_resources(project=project)
return _list_resources(
project=project,
start_timestamp=flask.request.args.get('start_timestamp'),
end_timestamp=flask.request.args.get('end_timestamp'),
)
@blueprint.route('/resources')

12
ceilometer/storage/base.py
View File

@ -75,7 +75,8 @@ class Connection(object):
"""
@abc.abstractmethod
def get_resources(self, user=None, project=None, source=None):
def get_resources(self, user=None, project=None, source=None,
start_timestamp=None, end_timestamp=None):
"""Return an iterable of dictionaries containing resource information.
{ 'resource_id': UUID of the resource,
@ -86,14 +87,17 @@ class Connection(object):
'meter': list of the meters reporting data for the resource,
}
:param user: Optional resource owner.
:param project: Optional resource owner.
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param source: Optional source filter.
:param start_timestamp: Optional modified timestamp start range.
:param end_timestamp: Optional modified timestamp end range.
"""
@abc.abstractmethod
def get_raw_events(self, event_filter):
"""Return an iterable of raw event data.
"""Return an iterable of raw event data as created by
:func:`ceilometer.meter.meter_message_from_counter`.
"""
@abc.abstractmethod

11
ceilometer/storage/impl_log.py
View File

@ -68,16 +68,21 @@ class Connection(base.Connection):
:param source: Optional source filter.
"""
def get_resources(self, user=None, project=None, source=None):
def get_resources(self, user=None, project=None, source=None,
start_timestamp=None, end_timestamp=None):
"""Return an iterable of tuples containing resource ids and
the most recent version of the metadata for the resource.
:param user: The event owner.
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param source: Optional source filter.
:param start_timestamp: Optional modified timestamp start range.
:param end_timestamp: Optional modified timestamp end range.
"""
def get_raw_events(self, event_filter):
"""Return an iterable of event data.
"""Return an iterable of raw event data as created by
:func:`ceilometer.meter.meter_message_from_counter`.
"""
def get_volume_sum(self, event_filter):

49
ceilometer/storage/impl_mongodb.py
View File

@ -83,6 +83,20 @@ class MongoDBStorage(base.StorageEngine):
return Connection(conf)
def make_timestamp_range(start, end):
"""Given two possible datetimes, create the query
document to find timestamps within that range
using $gte for the lower bound and $lt for the
upper bound.
"""
ts_range = {}
if start:
ts_range['$gte'] = start
if end:
ts_range['$lt'] = end
return ts_range
def make_query_from_filter(event_filter, require_meter=True):
"""Return a query dictionary based on the settings in the filter.
@ -102,10 +116,10 @@ def make_query_from_filter(event_filter, require_meter=True):
elif require_meter:
raise RuntimeError('Missing required meter specifier')
if event_filter.start:
q['timestamp'] = {'$gte': event_filter.start}
if event_filter.end:
q['timestamp'] = {'$lt': event_filter.end}
ts_range = make_timestamp_range(event_filter.start, event_filter.end)
if ts_range:
q['timestamp'] = ts_range
if event_filter.resource:
q['resource_id'] = event_filter.resource
if event_filter.source:
@ -263,7 +277,8 @@ class Connection(base.Connection):
q['source'] = source
return sorted(self.db.project.find(q).distinct('_id'))
def get_resources(self, user=None, project=None, source=None):
def get_resources(self, user=None, project=None, source=None,
start_timestamp=None, end_timestamp=None):
"""Return an iterable of dictionaries containing resource information.
{ 'resource_id': UUID of the resource,
@ -274,9 +289,11 @@ class Connection(base.Connection):
'meter': list of the meters reporting data for the resource,
}
:param user: Optional resource owner.
:param project: Optional resource owner.
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param source: Optional source filter.
:param start_timestamp: Optional modified timestamp start range.
:param end_timestamp: Optional modified timestamp end range.
"""
q = {}
if user is not None:
@ -285,6 +302,21 @@ class Connection(base.Connection):
q['project_id'] = project
if source is not None:
q['source'] = source
# FIXME(dhellmann): This may not perform very well,
# but doing any better will require changing the database
# schema and that will need more thought than I have time
# to put into it today.
if start_timestamp or end_timestamp:
# Look for resources matching the above criteria and with
# events in the time range we care about, then change the
# resource query to return just those resources by id.
ts_range = make_timestamp_range(start_timestamp, end_timestamp)
if ts_range:
q['timestamp'] = ts_range
resource_ids = self.db.meter.find(q).distinct('resource_id')
# Overwrite the query to just filter on the ids
# we have discovered to be interesting.
q = {'_id': {'$in': resource_ids}}
for resource in self.db.resource.find(q):
r = {}
r.update(resource)
@ -295,7 +327,8 @@ class Connection(base.Connection):
yield r
def get_raw_events(self, event_filter):
"""Return an iterable of event data.
"""Return an iterable of raw event data as created by
:func:`ceilometer.meter.meter_message_from_counter`.
"""
q = make_query_from_filter(event_filter, require_meter=False)
events = self.db.meter.find(q)

43
tests/storage/test_impl_mongodb.py
View File

@ -55,6 +55,8 @@ from ming import mim
import mox
from nose.plugins import skip
from nova import test
from ceilometer import counter
from ceilometer import meter
from ceilometer import storage
@ -63,14 +65,15 @@ from ceilometer.storage import impl_mongodb
LOG = logging.getLogger(__name__)
FORCING_MONGO = bool(int(os.environ.get('CEILOMETER_TEST_LIVE', 0)))
class Connection(impl_mongodb.Connection):
def _get_connection(self, conf):
# Use a real MongoDB server if we can connect, but fall back
# to a Mongo-in-memory connection if we cannot.
self.force_mongo = bool(int(os.environ.get('CEILOMETER_TEST_LIVE', 0)))
if self.force_mongo:
if FORCING_MONGO:
try:
return super(Connection, self)._get_connection(conf)
except:
@ -253,6 +256,31 @@ class ResourceTest(MongoDBEngineTestBase):
else:
assert False, 'Never found resource-id'
def test_get_resources_start_timestamp(self):
timestamp = datetime.datetime(2012, 7, 2, 10, 42)
resources = list(self.conn.get_resources(start_timestamp=timestamp))
resource_ids = [r['resource_id'] for r in resources]
expected = set(['resource-id-2', 'resource-id-3'])
assert set(resource_ids) == expected
def test_get_resources_end_timestamp(self):
timestamp = datetime.datetime(2012, 7, 2, 10, 42)
resources = list(self.conn.get_resources(end_timestamp=timestamp))
resource_ids = [r['resource_id'] for r in resources]
expected = set(['resource-id', 'resource-id-alternate'])
assert set(resource_ids) == expected
@test.skip_if(not FORCING_MONGO, 'Requires real MongoDB test database')
def test_get_resources_both_timestamps(self):
start_ts = datetime.datetime(2012, 7, 2, 10, 42)
end_ts = datetime.datetime(2012, 7, 2, 10, 43)
resources = list(self.conn.get_resources(start_timestamp=start_ts,
end_timestamp=end_ts)
)
resource_ids = [r['resource_id'] for r in resources]
expected = set(['resource-id-2'])
assert set(resource_ids) == expected
def test_get_resources_by_source(self):
resources = list(self.conn.get_resources(source='test-1'))
assert len(resources) == 1
@ -321,6 +349,17 @@ class MeterTest(MongoDBEngineTestBase):
assert length == 1
assert results[0]['timestamp'] == datetime.datetime(2012, 7, 2, 10, 40)
@test.skip_if(not FORCING_MONGO, 'Requires real MongoDB test database')
def test_get_raw_events_by_both_times(self):
f = storage.EventFilter(
start=datetime.datetime(2012, 7, 2, 10, 42),
end=datetime.datetime(2012, 7, 2, 10, 43),
)
results = list(self.conn.get_raw_events(f))
length = len(results)
assert length == 1
assert results[0]['timestamp'] == datetime.datetime(2012, 7, 2, 10, 42)
def test_get_raw_events_by_meter(self):
f = storage.EventFilter(
user='user-id',