From 7194e92568756775b7baf5e996f1b378e004695e Mon Sep 17 00:00:00 2001 From: Mehdi Abaakouk Date: Mon, 16 Dec 2013 19:06:55 +0100 Subject: [PATCH] Convert event timestamp to PrecisionTimestamp Before the timestamps for event and trait are stored in a Float In mysql, the default precision won't work well, this one are rounded. This patch, change the Float by the PrecisionTimestamp Column Type, like the sample timestamp that demand the same precision. And ensure that get_events always return datetime object. Change-Id: I1f986784b07afa50bdbb7faa9499e7b614da8ec1 --- ceilometer/storage/impl_sqlalchemy.py | 18 ++--- .../024_event_use_floatingprecision.py | 81 +++++++++++++++++++ ceilometer/storage/sqlalchemy/models.py | 6 +- .../tests/storage/test_impl_sqlalchemy.py | 3 +- .../tests/storage/test_storage_scenarios.py | 20 ++++- 5 files changed, 109 insertions(+), 19 deletions(-) create mode 100644 ceilometer/storage/sqlalchemy/migrate_repo/versions/024_event_use_floatingprecision.py diff --git a/ceilometer/storage/impl_sqlalchemy.py b/ceilometer/storage/impl_sqlalchemy.py index 97d1666ba..c57b1fd7b 100644 --- a/ceilometer/storage/impl_sqlalchemy.py +++ b/ceilometer/storage/impl_sqlalchemy.py @@ -863,8 +863,6 @@ class Connection(base.Connection): values = {'t_string': None, 't_float': None, 't_int': None, 't_datetime': None} value = trait_model.value - if trait_model.dtype == api_models.Trait.DATETIME_TYPE: - value = utils.dt_to_decimal(value) values[value_map[trait_model.dtype]] = value return models.Trait(trait_type, event, **values) @@ -893,8 +891,8 @@ class Connection(base.Connection): event_type = cls._get_or_create_event_type(event_model.event_type, session=session) - generated = utils.dt_to_decimal(event_model.generated) - event = models.Event(event_model.message_id, event_type, generated) + event = models.Event(event_model.message_id, event_type, + event_model.generated) session.add(event) new_traits = [] @@ -944,8 +942,8 @@ class Connection(base.Connection): :param event_filter: EventFilter instance """ - start = utils.dt_to_decimal(event_filter.start_time) - end = utils.dt_to_decimal(event_filter.end_time) + start = event_filter.start_time + end = event_filter.end_time session = sqlalchemy_session.get_session() with session.begin(): event_query = session.query(models.Event) @@ -992,8 +990,7 @@ class Connection(base.Connection): elif key == 't_int': conditions.append(models.Trait.t_int == value) elif key == 't_datetime': - dt = utils.dt_to_decimal(value) - conditions.append(models.Trait.t_datetime == dt) + conditions.append(models.Trait.t_datetime == value) elif key == 't_float': conditions.append(models.Trait.t_float == value) @@ -1030,11 +1027,10 @@ class Connection(base.Connection): for trait in query.all(): event = event_models_dict.get(trait.event_id) if not event: - generated = utils.decimal_to_dt(trait.event.generated) event = api_models.Event( trait.event.message_id, trait.event.event_type.desc, - generated, []) + trait.event.generated, []) event_models_dict[trait.event_id] = event trait_model = api_models.Trait(trait.trait_type.desc, trait.trait_type.data_type, @@ -1111,7 +1107,7 @@ class Connection(base.Connection): .join(models.EventType, and_(models.EventType.id == models.Event.event_type_id, - models.EventType.desc == event_type))) + models.EventType.desc == event_type))) for trait in query.all(): type = trait.trait_type diff --git a/ceilometer/storage/sqlalchemy/migrate_repo/versions/024_event_use_floatingprecision.py b/ceilometer/storage/sqlalchemy/migrate_repo/versions/024_event_use_floatingprecision.py new file mode 100644 index 000000000..e102c6947 --- /dev/null +++ b/ceilometer/storage/sqlalchemy/migrate_repo/versions/024_event_use_floatingprecision.py @@ -0,0 +1,81 @@ +# -*- encoding: utf-8 -*- +# +# Copyright © 2013 eNovance SAS +# +# Author: Mehdi Abaakouk +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy as sa + +from ceilometer.storage.sqlalchemy import models + + +def _paged(query, size): + offset = 0 + while True: + page = query.offset(offset).limit(size).execute() + if page.rowcount <= 0: + # There are no more rows + break + for row in page: + yield row + offset += size + + +def _convert_data_type(table, col, from_t, to_t, pk_attr='id', index=False): + temp_col_n = 'convert_data_type_temp_col' + # Override column we're going to convert with from_t, since the type we're + # replacing could be custom and we need to tell SQLALchemy how to perform + # CRUD operations with it. + table = sa.Table(table.name, table.metadata, sa.Column(col, from_t), + extend_existing=True) + sa.Column(temp_col_n, to_t).create(table) + + key_attr = getattr(table.c, pk_attr) + orig_col = getattr(table.c, col) + new_col = getattr(table.c, temp_col_n) + + query = sa.select([key_attr, orig_col]) + for key, value in _paged(query, 1000): + table.update().where(key_attr == key)\ + .values({temp_col_n: value}).execute() + + orig_col.drop() + new_col.alter(name=col) + if index: + sa.Index('ix_%s_%s' % (table.name, col), new_col).create() + + +def upgrade(migrate_engine): + if migrate_engine.name == 'mysql': + meta = sa.MetaData(bind=migrate_engine) + event = sa.Table('event', meta, autoload=True) + _convert_data_type(event, 'generated', sa.Float(), + models.PreciseTimestamp(), + pk_attr='id', index=True) + trait = sa.Table('trait', meta, autoload=True) + _convert_data_type(trait, 't_datetime', sa.Float(), + models.PreciseTimestamp(), + pk_attr='id', index=True) + + +def downgrade(migrate_engine): + if migrate_engine.name == 'mysql': + meta = sa.MetaData(bind=migrate_engine) + event = sa.Table('event', meta, autoload=True) + _convert_data_type(event, 'generated', models.PreciseTimestamp(), + sa.Float(), pk_attr='id', index=True) + trait = sa.Table('trait', meta, autoload=True) + _convert_data_type(trait, 't_datetime', models.PreciseTimestamp(), + sa.Float(), pk_attr='id', index=True) diff --git a/ceilometer/storage/sqlalchemy/models.py b/ceilometer/storage/sqlalchemy/models.py index ed7cd0066..d3d3f2a1b 100644 --- a/ceilometer/storage/sqlalchemy/models.py +++ b/ceilometer/storage/sqlalchemy/models.py @@ -320,7 +320,7 @@ class Event(Base): ) id = Column(Integer, primary_key=True) message_id = Column(String(50), unique=True) - generated = Column(Float(asdecimal=True)) + generated = Column(PreciseTimestamp()) event_type_id = Column(Integer, ForeignKey('event_type.id')) event_type = relationship("EventType", backref=backref('event_type')) @@ -379,7 +379,7 @@ class Trait(Base): t_string = Column(String(255), nullable=True, default=None) t_float = Column(Float, nullable=True, default=None) t_int = Column(Integer, nullable=True, default=None) - t_datetime = Column(Float(asdecimal=True), nullable=True, default=None) + t_datetime = Column(PreciseTimestamp(), nullable=True, default=None) event_id = Column(Integer, ForeignKey('event.id')) event = relationship("Event", backref=backref('event', order_by=id)) @@ -409,7 +409,7 @@ class Trait(Base): if dtype == api_models.Trait.FLOAT_TYPE: return self.t_float if dtype == api_models.Trait.DATETIME_TYPE: - return utils.decimal_to_dt(self.t_datetime) + return self.t_datetime if dtype == api_models.Trait.TEXT_TYPE: return self.t_string diff --git a/ceilometer/tests/storage/test_impl_sqlalchemy.py b/ceilometer/tests/storage/test_impl_sqlalchemy.py index f8f0ec283..2d72b8e91 100644 --- a/ceilometer/tests/storage/test_impl_sqlalchemy.py +++ b/ceilometer/tests/storage/test_impl_sqlalchemy.py @@ -34,7 +34,6 @@ from ceilometer.storage import models from ceilometer.storage.sqlalchemy import models as sql_models from ceilometer.tests import db as tests_db from ceilometer.tests.storage import test_storage_scenarios as scenarios -from ceilometer import utils class EventTestBase(tests_db.TestBase): @@ -147,7 +146,7 @@ class EventTest(EventTestBase): self.assertIsNone(trait.t_int) self.assertIsNone(trait.t_string) self.assertIsNone(trait.t_float) - self.assertEqual(trait.t_datetime, utils.dt_to_decimal(now)) + self.assertEqual(trait.t_datetime, now) self.assertIsNotNone(trait.trait_type.desc) def test_bad_event(self): diff --git a/ceilometer/tests/storage/test_storage_scenarios.py b/ceilometer/tests/storage/test_storage_scenarios.py index 448baa087..d08ca87d1 100644 --- a/ceilometer/tests/storage/test_storage_scenarios.py +++ b/ceilometer/tests/storage/test_storage_scenarios.py @@ -2145,7 +2145,7 @@ class EventTest(EventTestBase): class GetEventTest(EventTestBase): def prepare_data(self): - event_models = [] + self.event_models = [] base = 0 self.start = datetime.datetime(2013, 12, 31, 5, 0) now = self.start @@ -2160,14 +2160,28 @@ class GetEventTest(EventTestBase): ('trait_C', models.Trait.FLOAT_TYPE, float(base) + 0.123456), ('trait_D', models.Trait.DATETIME_TYPE, now)]] - event_models.append( + self.event_models.append( models.Event("id_%s_%d" % (event_type, base), event_type, now, trait_models)) base += 100 now = now + datetime.timedelta(hours=1) self.end = now - self.conn.record_events(event_models) + self.conn.record_events(self.event_models) + + def test_generated_is_datetime(self): + event_filter = storage.EventFilter(self.start, self.end) + events = self.conn.get_events(event_filter) + self.assertEqual(6, len(events)) + for i, event in enumerate(events): + self.assertTrue(isinstance(event.generated, datetime.datetime)) + self.assertEqual(event.generated, + self.event_models[i].generated) + model_traits = self.event_models[i].traits + for j, trait in enumerate(event.traits): + if trait.dtype == models.Trait.DATETIME_TYPE: + self.assertTrue(isinstance(trait.value, datetime.datetime)) + self.assertEqual(trait.value, model_traits[j].value) def test_simple_get(self): event_filter = storage.EventFilter(self.start, self.end)