diff --git a/ceilometer/api/controllers/v2.py b/ceilometer/api/controllers/v2.py index 677f3eeec..dd6fc19b1 100644 --- a/ceilometer/api/controllers/v2.py +++ b/ceilometer/api/controllers/v2.py @@ -171,16 +171,12 @@ def _query_to_kwargs(query, db_func): metaquery = {} for i in query: if i.field == 'timestamp': - # FIXME(dhellmann): This logic is not consistent with the - # way the timestamps are treated inside the mongo driver - # (the end timestamp is always tested using $lt). We - # should just pass a single timestamp through to the - # storage layer with the operator and let the storage - # layer use that operator. if i.op in ('lt', 'le'): stamp['end_timestamp'] = i.value + stamp['end_timestamp_op'] = i.op elif i.op in ('gt', 'ge'): stamp['start_timestamp'] = i.value + stamp['start_timestamp_op'] = i.op else: LOG.warn('_query_to_kwargs ignoring %r unexpected op %r"' % (i.field, i.op)) @@ -209,6 +205,10 @@ def _query_to_kwargs(query, db_func): else: raise wsme.exc.UnknownArgument('timestamp', "not valid for this resource") + if 'start_timestamp_op' in stamp: + kwargs['start_timestamp_op'] = stamp['start_timestamp_op'] + if 'end_timestamp_op' in stamp: + kwargs['end_timestamp_op'] = stamp['end_timestamp_op'] if trans: for k in trans: diff --git a/ceilometer/storage/__init__.py b/ceilometer/storage/__init__.py index ca273f768..9751df586 100644 --- a/ceilometer/storage/__init__.py +++ b/ceilometer/storage/__init__.py @@ -81,19 +81,26 @@ class SampleFilter(object): :param user: The sample owner. :param project: The sample project. - :param start: Earliest timestamp to include. - :param end: Only include samples with timestamp less than this. + :param start: Earliest time point in the request. + :param start_timestamp_op: Earliest timestamp operation in the request. + :param end: Latest time point in the request. + :param end_timestamp_op: Latest timestamp operation in the request. :param resource: Optional filter for resource id. :param meter: Optional filter for meter type using the meter name. :param source: Optional source filter. :param metaquery: Optional filter on the metadata """ - def __init__(self, user=None, project=None, start=None, end=None, - resource=None, meter=None, source=None, metaquery={}): + def __init__(self, user=None, project=None, + start=None, start_timestamp_op=None, + end=None, end_timestamp_op=None, + resource=None, meter=None, + source=None, metaquery={}): self.user = user self.project = project self.start = utils.sanitize_timestamp(start) + self.start_timestamp_op = start_timestamp_op self.end = utils.sanitize_timestamp(end) + self.end_timestamp_op = end_timestamp_op self.resource = resource self.meter = meter self.source = source diff --git a/ceilometer/storage/base.py b/ceilometer/storage/base.py index 9e3d1e76f..a7d80b842 100644 --- a/ceilometer/storage/base.py +++ b/ceilometer/storage/base.py @@ -98,7 +98,8 @@ class Connection(object): @abc.abstractmethod def get_resources(self, user=None, project=None, source=None, - start_timestamp=None, end_timestamp=None, + start_timestamp=None, start_timestamp_op=None, + end_timestamp=None, end_timestamp_op=None, metaquery={}, resource=None): """Return an iterable of models.Resource instances containing resource information. @@ -107,7 +108,9 @@ class Connection(object): :param project: Optional ID for project that owns the resource. :param source: Optional source filter. :param start_timestamp: Optional modified timestamp start range. + :param start_timestamp_op: Optional timestamp start range operation. :param end_timestamp: Optional modified timestamp end range. + :param end_timestamp_op: Optional timestamp end range operation. :param metaquery: Optional dict with metadata to match on. :param resource: Optional resource filter. """ diff --git a/ceilometer/storage/impl_hbase.py b/ceilometer/storage/impl_hbase.py index d7b5d5882..e80d19fb5 100644 --- a/ceilometer/storage/impl_hbase.py +++ b/ceilometer/storage/impl_hbase.py @@ -278,7 +278,8 @@ class Connection(base.Connection): return (key for key, ignored in self.project.scan(**scan_args)) def get_resources(self, user=None, project=None, source=None, - start_timestamp=None, end_timestamp=None, + start_timestamp=None, start_timestamp_op=None, + end_timestamp=None, end_timestamp_op=None, metaquery={}): """Return an iterable of models.Resource instances @@ -286,7 +287,9 @@ class Connection(base.Connection): :param project: Optional ID for project that owns the resource. :param source: Optional source filter. :param start_timestamp: Optional modified timestamp start range. + :param start_timestamp_op: Optional start time operator, like ge, gt. :param end_timestamp: Optional modified timestamp end range. + :param end_timestamp_op: Optional end time operator, like lt, le. :param metaquery: Optional dict with metadata to match on. """ def make_resource(data): @@ -313,7 +316,9 @@ class Connection(base.Connection): project=project, source=source, start=start_timestamp, + start_op=start_timestamp_op, end=end_timestamp, + end_op=end_timestamp_op, require_meter=False, query_only=False) LOG.debug("Query Meter table: %s" % q) @@ -692,8 +697,8 @@ def reverse_timestamp(dt): def make_query(user=None, project=None, meter=None, - resource=None, source=None, start=None, end=None, - require_meter=True, query_only=False): + resource=None, source=None, start=None, start_op=None, + end=None, end_op=None, require_meter=True, query_only=False): """Return a filter query string based on the selected parameters. :param user: Optional user-id @@ -702,7 +707,9 @@ def make_query(user=None, project=None, meter=None, :param resource: Optional resource-id :param source: Optional source-id :param start: Optional start timestamp + :param start_op: Optional start timestamp operator, like gt, ge :param end: Optional end timestamp + :param end_op: Optional end timestamp operator, like lt, le :param require_meter: If true and the filter does not have a meter, raise an error. :param query_only: If true only returns the filter query, @@ -727,6 +734,12 @@ def make_query(user=None, project=None, meter=None, rts_start = str(reverse_timestamp(start) + 1) if start else "" rts_end = str(reverse_timestamp(end) + 1) if end else "" + #By default, we are using ge for lower bound and lt for upper bound + if start_op == 'gt': + rts_start = str(long(rts_start) - 2) + if end_op == 'le': + rts_end = str(long(rts_end) - 1) + # when start_time and end_time is provided, # if it's filtered by meter, # rowkey will be used in the query; @@ -765,7 +778,10 @@ def make_query_from_filter(sample_filter, require_meter=True): return make_query(sample_filter.user, sample_filter.project, sample_filter.meter, sample_filter.resource, sample_filter.source, sample_filter.start, - sample_filter.end, require_meter) + sample_filter.start_timestamp_op, + sample_filter.end, + sample_filter.end_timestamp_op, + require_meter) def _make_rowkey_scan(meter, rts_start=None, rts_end=None): diff --git a/ceilometer/storage/impl_log.py b/ceilometer/storage/impl_log.py index 418b0f6af..1c3ffa68b 100644 --- a/ceilometer/storage/impl_log.py +++ b/ceilometer/storage/impl_log.py @@ -77,7 +77,8 @@ class Connection(base.Connection): return [] def get_resources(self, user=None, project=None, source=None, - start_timestamp=None, end_timestamp=None, + start_timestamp=None, start_timestamp_op=None, + end_timestamp=None, end_timestamp_op=None, metaquery={}, resource=None): """Return an iterable of dictionaries containing resource information. @@ -93,7 +94,9 @@ class Connection(base.Connection): :param project: Optional ID for project that owns the resource. :param source: Optional source filter. :param start_timestamp: Optional modified timestamp start range. + :param start_timestamp_op: Optional start time operator, like gt, ge. :param end_timestamp: Optional modified timestamp end range. + :param end_timestamp_op: Optional end time operator, like lt, le. :param metaquery: Optional dict with metadata to match on. :param resource: Optional resource filter. """ diff --git a/ceilometer/storage/impl_mongodb.py b/ceilometer/storage/impl_mongodb.py index 2ea792436..cb9ac8b1c 100644 --- a/ceilometer/storage/impl_mongodb.py +++ b/ceilometer/storage/impl_mongodb.py @@ -93,17 +93,28 @@ class MongoDBStorage(base.StorageEngine): return Connection(conf) -def make_timestamp_range(start, end): - """Given two possible datetimes, create the query - document to find timestamps within that range - using $gte for the lower bound and $lt for the +def make_timestamp_range(start, end, + start_timestamp_op=None, end_timestamp_op=None): + """Given two possible datetimes and their operations, create the query + document to find timestamps within that range. + By default, using $gte for the lower bound and $lt for the upper bound. """ ts_range = {} + if start: - ts_range['$gte'] = start + if start_timestamp_op == 'gt': + start_timestamp_op = '$gt' + else: + start_timestamp_op = '$gte' + ts_range[start_timestamp_op] = start + if end: - ts_range['$lt'] = end + if end_timestamp_op == 'le': + end_timestamp_op = '$lte' + else: + end_timestamp_op = '$lt' + ts_range[end_timestamp_op] = end return ts_range @@ -126,7 +137,9 @@ def make_query_from_filter(sample_filter, require_meter=True): elif require_meter: raise RuntimeError('Missing required meter specifier') - ts_range = make_timestamp_range(sample_filter.start, sample_filter.end) + ts_range = make_timestamp_range(sample_filter.start, sample_filter.end, + sample_filter.start_timestamp_op, + sample_filter.end_timestamp_op) if ts_range: q['timestamp'] = ts_range @@ -355,7 +368,8 @@ class Connection(base.Connection): return sorted(self.db.project.find(q).distinct('_id')) def get_resources(self, user=None, project=None, source=None, - start_timestamp=None, end_timestamp=None, + start_timestamp=None, start_timestamp_op=None, + end_timestamp=None, end_timestamp_op=None, metaquery={}, resource=None): """Return an iterable of models.Resource instances @@ -363,7 +377,9 @@ class Connection(base.Connection): :param project: Optional ID for project that owns the resource. :param source: Optional source filter. :param start_timestamp: Optional modified timestamp start range. + :param start_timestamp_op: Optional start time operator, like gt, ge. :param end_timestamp: Optional modified timestamp end range. + :param end_timestamp_op: Optional end time operator, like lt, le. :param metaquery: Optional dict with metadata to match on. :param resource: Optional resource filter. """ @@ -388,7 +404,9 @@ class Connection(base.Connection): # Look for resources matching the above criteria and with # samples in the time range we care about, then change the # resource query to return just those resources by id. - ts_range = make_timestamp_range(start_timestamp, end_timestamp) + ts_range = make_timestamp_range(start_timestamp, end_timestamp, + start_timestamp_op, + end_timestamp_op) if ts_range: q['timestamp'] = ts_range diff --git a/ceilometer/storage/impl_sqlalchemy.py b/ceilometer/storage/impl_sqlalchemy.py index 340c09deb..b31b5daa5 100644 --- a/ceilometer/storage/impl_sqlalchemy.py +++ b/ceilometer/storage/impl_sqlalchemy.py @@ -119,10 +119,16 @@ def make_query_from_filter(query, sample_filter, require_meter=True): query = query.filter(Meter.sources.any(id=sample_filter.source)) if sample_filter.start: ts_start = sample_filter.start - query = query.filter(Meter.timestamp >= ts_start) + if sample_filter.start_timestamp_op == 'gt': + query = query.filter(Meter.timestamp > ts_start) + else: + query = query.filter(Meter.timestamp >= ts_start) if sample_filter.end: ts_end = sample_filter.end - query = query.filter(Meter.timestamp < ts_end) + if sample_filter.end_timestamp_op == 'le': + query = query.filter(Meter.timestamp <= ts_end) + else: + query = query.filter(Meter.timestamp < ts_end) if sample_filter.user: query = query.filter_by(user_id=sample_filter.user) if sample_filter.project: @@ -240,7 +246,8 @@ class Connection(base.Connection): @staticmethod def get_resources(user=None, project=None, source=None, - start_timestamp=None, end_timestamp=None, + start_timestamp=None, start_timestamp_op=None, + end_timestamp=None, end_timestamp_op=None, metaquery={}, resource=None): """Return an iterable of api_models.Resource instances @@ -248,7 +255,9 @@ class Connection(base.Connection): :param project: Optional ID for project that owns the resource. :param source: Optional source filter. :param start_timestamp: Optional modified timestamp start range. + :param start_timestamp_op: Optonal start time operator, like gt, ge. :param end_timestamp: Optional modified timestamp end range. + :param end_timestamp_op: Optional end time operator, like lt, le. :param metaquery: Optional dict with metadata to match on. :param resource: Optional resource filter. """ @@ -259,9 +268,15 @@ class Connection(base.Connection): if source is not None: query = query.filter(Meter.sources.any(id=source)) if start_timestamp: - query = query.filter(Meter.timestamp >= start_timestamp) + if start_timestamp_op == 'gt': + query = query.filter(Meter.timestamp > start_timestamp) + else: + query = query.filter(Meter.timestamp >= start_timestamp) if end_timestamp: - query = query.filter(Meter.timestamp < end_timestamp) + if end_timestamp_op == 'le': + query = query.filter(Meter.timestamp <= end_timestamp) + else: + query = query.filter(Meter.timestamp < end_timestamp) if project is not None: query = query.filter(Meter.project_id == project) if resource is not None: diff --git a/tests/storage/base.py b/tests/storage/base.py index 53fce288c..a9f87b5a3 100644 --- a/tests/storage/base.py +++ b/tests/storage/base.py @@ -193,28 +193,81 @@ class ResourceTest(DBTestBase): def test_get_resources_start_timestamp(self): timestamp = datetime.datetime(2012, 7, 2, 10, 42) - resources = list(self.conn.get_resources(start_timestamp=timestamp)) - resource_ids = [r.resource_id for r in resources] expected = set(['resource-id-2', 'resource-id-3', 'resource-id-4', 'resource-id-6', 'resource-id-8']) + + resources = list(self.conn.get_resources(start_timestamp=timestamp)) + resource_ids = [r.resource_id for r in resources] + self.assertEqual(set(resource_ids), expected) + + resources = list(self.conn.get_resources(start_timestamp=timestamp, + start_timestamp_op='ge')) + resource_ids = [r.resource_id for r in resources] + self.assertEqual(set(resource_ids), expected) + + resources = list(self.conn.get_resources(start_timestamp=timestamp, + start_timestamp_op='gt')) + resource_ids = [r.resource_id for r in resources] + expected.remove('resource-id-2') self.assertEqual(set(resource_ids), expected) def test_get_resources_end_timestamp(self): timestamp = datetime.datetime(2012, 7, 2, 10, 42) - resources = list(self.conn.get_resources(end_timestamp=timestamp)) - resource_ids = [r.resource_id for r in resources] expected = set(['resource-id', 'resource-id-alternate', 'resource-id-5', 'resource-id-7']) + + resources = list(self.conn.get_resources(end_timestamp=timestamp)) + resource_ids = [r.resource_id for r in resources] + self.assertEqual(set(resource_ids), expected) + + resources = list(self.conn.get_resources(end_timestamp=timestamp, + end_timestamp_op='lt')) + resource_ids = [r.resource_id for r in resources] + self.assertEqual(set(resource_ids), expected) + + resources = list(self.conn.get_resources(end_timestamp=timestamp, + end_timestamp_op='le')) + resource_ids = [r.resource_id for r in resources] + expected.add('resource-id-2') self.assertEqual(set(resource_ids), expected) def test_get_resources_both_timestamps(self): start_ts = datetime.datetime(2012, 7, 2, 10, 42) end_ts = datetime.datetime(2012, 7, 2, 10, 43) + resources = list(self.conn.get_resources(start_timestamp=start_ts, end_timestamp=end_ts)) resource_ids = [r.resource_id for r in resources] assert set(resource_ids) == set(['resource-id-2']) + resources = list(self.conn.get_resources(start_timestamp=start_ts, + end_timestamp=end_ts, + start_timestamp_op='ge', + end_timestamp_op='lt')) + resource_ids = [r.resource_id for r in resources] + assert set(resource_ids) == set(['resource-id-2']) + + resources = list(self.conn.get_resources(start_timestamp=start_ts, + end_timestamp=end_ts, + start_timestamp_op='gt', + end_timestamp_op='lt')) + resource_ids = [r.resource_id for r in resources] + assert len(resource_ids) == 0 + + resources = list(self.conn.get_resources(start_timestamp=start_ts, + end_timestamp=end_ts, + start_timestamp_op='gt', + end_timestamp_op='le')) + resource_ids = [r.resource_id for r in resources] + assert set(resource_ids) == set(['resource-id-3']) + + resources = list(self.conn.get_resources(start_timestamp=start_ts, + end_timestamp=end_ts, + start_timestamp_op='ge', + end_timestamp_op='le')) + resource_ids = [r.resource_id for r in resources] + assert set(resource_ids) == set(['resource-id-2', 'resource-id-3']) + def test_get_resources_by_source(self): resources = list(self.conn.get_resources(source='test-1')) assert len(resources) == 1 @@ -353,33 +406,79 @@ class RawSampleTest(DBTestBase): self.assertTrue(got_not_imp) def test_get_samples_by_start_time(self): + timestamp = datetime.datetime(2012, 7, 2, 10, 41) f = storage.SampleFilter( user='user-id', - start=datetime.datetime(2012, 7, 2, 10, 41), + start=timestamp, ) + results = list(self.conn.get_samples(f)) assert len(results) == 1 - assert results[0].timestamp == datetime.datetime(2012, 7, 2, 10, 41) + assert results[0].timestamp == timestamp + + f.start_timestamp_op = 'ge' + results = list(self.conn.get_samples(f)) + assert len(results) == 1 + assert results[0].timestamp == timestamp + + f.start_timestamp_op = 'gt' + results = list(self.conn.get_samples(f)) + assert len(results) == 0 def test_get_samples_by_end_time(self): + timestamp = datetime.datetime(2012, 7, 2, 10, 40) f = storage.SampleFilter( user='user-id', - end=datetime.datetime(2012, 7, 2, 10, 41), + end=timestamp, ) + results = list(self.conn.get_samples(f)) - length = len(results) - assert length == 1 - assert results[0].timestamp == datetime.datetime(2012, 7, 2, 10, 40) + assert len(results) == 0 + + f.end_timestamp_op = 'lt' + results = list(self.conn.get_samples(f)) + assert len(results) == 0 + + f.end_timestamp_op = 'le' + results = list(self.conn.get_samples(f)) + assert len(results) == 1 + assert results[0].timestamp == timestamp def test_get_samples_by_both_times(self): + start_ts = datetime.datetime(2012, 7, 2, 10, 42) + end_ts = datetime.datetime(2012, 7, 2, 10, 43) f = storage.SampleFilter( - start=datetime.datetime(2012, 7, 2, 10, 42), - end=datetime.datetime(2012, 7, 2, 10, 43), + start=start_ts, + end=end_ts, ) + results = list(self.conn.get_samples(f)) - length = len(results) - assert length == 1 - assert results[0].timestamp == datetime.datetime(2012, 7, 2, 10, 42) + assert len(results) == 1 + assert results[0].timestamp == start_ts + + f.start_timestamp_op = 'gt' + f.end_timestamp_op = 'lt' + results = list(self.conn.get_samples(f)) + assert len(results) == 0 + + f.start_timestamp_op = 'ge' + f.end_timestamp_op = 'lt' + results = list(self.conn.get_samples(f)) + assert len(results) == 1 + assert results[0].timestamp == start_ts + + f.start_timestamp_op = 'gt' + f.end_timestamp_op = 'le' + results = list(self.conn.get_samples(f)) + assert len(results) == 1 + assert results[0].timestamp == end_ts + + f.start_timestamp_op = 'ge' + f.end_timestamp_op = 'le' + results = list(self.conn.get_samples(f)) + assert len(results) == 2 + assert results[0].timestamp == end_ts + assert results[1].timestamp == start_ts def test_get_samples_by_name(self): f = storage.SampleFilter(user='user-id', meter='no-such-meter')