diff --git a/ceilometer/storage/impl_sqlalchemy.py b/ceilometer/storage/impl_sqlalchemy.py index d628f6059..51cdea004 100644 --- a/ceilometer/storage/impl_sqlalchemy.py +++ b/ceilometer/storage/impl_sqlalchemy.py @@ -867,8 +867,6 @@ class Connection(base.Connection): values = {'t_string': None, 't_float': None, 't_int': None, 't_datetime': None} value = trait_model.value - if trait_model.dtype == api_models.Trait.DATETIME_TYPE: - value = utils.dt_to_decimal(value) values[value_map[trait_model.dtype]] = value return models.Trait(trait_type, event, **values) @@ -897,8 +895,8 @@ class Connection(base.Connection): event_type = cls._get_or_create_event_type(event_model.event_type, session=session) - generated = utils.dt_to_decimal(event_model.generated) - event = models.Event(event_model.message_id, event_type, generated) + event = models.Event(event_model.message_id, event_type, + event_model.generated) session.add(event) new_traits = [] @@ -948,8 +946,8 @@ class Connection(base.Connection): :param event_filter: EventFilter instance """ - start = utils.dt_to_decimal(event_filter.start_time) - end = utils.dt_to_decimal(event_filter.end_time) + start = event_filter.start_time + end = event_filter.end_time session = sqlalchemy_session.get_session() with session.begin(): event_query = session.query(models.Event) @@ -996,8 +994,7 @@ class Connection(base.Connection): elif key == 't_int': conditions.append(models.Trait.t_int == value) elif key == 't_datetime': - dt = utils.dt_to_decimal(value) - conditions.append(models.Trait.t_datetime == dt) + conditions.append(models.Trait.t_datetime == value) elif key == 't_float': conditions.append(models.Trait.t_float == value) @@ -1034,11 +1031,10 @@ class Connection(base.Connection): for trait in query.all(): event = event_models_dict.get(trait.event_id) if not event: - generated = utils.decimal_to_dt(trait.event.generated) event = api_models.Event( trait.event.message_id, trait.event.event_type.desc, - generated, []) + trait.event.generated, []) event_models_dict[trait.event_id] = event trait_model = api_models.Trait(trait.trait_type.desc, trait.trait_type.data_type, @@ -1115,7 +1111,7 @@ class Connection(base.Connection): .join(models.EventType, and_(models.EventType.id == models.Event.event_type_id, - models.EventType.desc == event_type))) + models.EventType.desc == event_type))) for trait in query.all(): type = trait.trait_type diff --git a/ceilometer/storage/sqlalchemy/migrate_repo/versions/024_event_use_floatingprecision.py b/ceilometer/storage/sqlalchemy/migrate_repo/versions/024_event_use_floatingprecision.py new file mode 100644 index 000000000..e102c6947 --- /dev/null +++ b/ceilometer/storage/sqlalchemy/migrate_repo/versions/024_event_use_floatingprecision.py @@ -0,0 +1,81 @@ +# -*- encoding: utf-8 -*- +# +# Copyright © 2013 eNovance SAS +# +# Author: Mehdi Abaakouk +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy as sa + +from ceilometer.storage.sqlalchemy import models + + +def _paged(query, size): + offset = 0 + while True: + page = query.offset(offset).limit(size).execute() + if page.rowcount <= 0: + # There are no more rows + break + for row in page: + yield row + offset += size + + +def _convert_data_type(table, col, from_t, to_t, pk_attr='id', index=False): + temp_col_n = 'convert_data_type_temp_col' + # Override column we're going to convert with from_t, since the type we're + # replacing could be custom and we need to tell SQLALchemy how to perform + # CRUD operations with it. + table = sa.Table(table.name, table.metadata, sa.Column(col, from_t), + extend_existing=True) + sa.Column(temp_col_n, to_t).create(table) + + key_attr = getattr(table.c, pk_attr) + orig_col = getattr(table.c, col) + new_col = getattr(table.c, temp_col_n) + + query = sa.select([key_attr, orig_col]) + for key, value in _paged(query, 1000): + table.update().where(key_attr == key)\ + .values({temp_col_n: value}).execute() + + orig_col.drop() + new_col.alter(name=col) + if index: + sa.Index('ix_%s_%s' % (table.name, col), new_col).create() + + +def upgrade(migrate_engine): + if migrate_engine.name == 'mysql': + meta = sa.MetaData(bind=migrate_engine) + event = sa.Table('event', meta, autoload=True) + _convert_data_type(event, 'generated', sa.Float(), + models.PreciseTimestamp(), + pk_attr='id', index=True) + trait = sa.Table('trait', meta, autoload=True) + _convert_data_type(trait, 't_datetime', sa.Float(), + models.PreciseTimestamp(), + pk_attr='id', index=True) + + +def downgrade(migrate_engine): + if migrate_engine.name == 'mysql': + meta = sa.MetaData(bind=migrate_engine) + event = sa.Table('event', meta, autoload=True) + _convert_data_type(event, 'generated', models.PreciseTimestamp(), + sa.Float(), pk_attr='id', index=True) + trait = sa.Table('trait', meta, autoload=True) + _convert_data_type(trait, 't_datetime', models.PreciseTimestamp(), + sa.Float(), pk_attr='id', index=True) diff --git a/ceilometer/storage/sqlalchemy/models.py b/ceilometer/storage/sqlalchemy/models.py index ed7cd0066..d3d3f2a1b 100644 --- a/ceilometer/storage/sqlalchemy/models.py +++ b/ceilometer/storage/sqlalchemy/models.py @@ -320,7 +320,7 @@ class Event(Base): ) id = Column(Integer, primary_key=True) message_id = Column(String(50), unique=True) - generated = Column(Float(asdecimal=True)) + generated = Column(PreciseTimestamp()) event_type_id = Column(Integer, ForeignKey('event_type.id')) event_type = relationship("EventType", backref=backref('event_type')) @@ -379,7 +379,7 @@ class Trait(Base): t_string = Column(String(255), nullable=True, default=None) t_float = Column(Float, nullable=True, default=None) t_int = Column(Integer, nullable=True, default=None) - t_datetime = Column(Float(asdecimal=True), nullable=True, default=None) + t_datetime = Column(PreciseTimestamp(), nullable=True, default=None) event_id = Column(Integer, ForeignKey('event.id')) event = relationship("Event", backref=backref('event', order_by=id)) @@ -409,7 +409,7 @@ class Trait(Base): if dtype == api_models.Trait.FLOAT_TYPE: return self.t_float if dtype == api_models.Trait.DATETIME_TYPE: - return utils.decimal_to_dt(self.t_datetime) + return self.t_datetime if dtype == api_models.Trait.TEXT_TYPE: return self.t_string diff --git a/ceilometer/tests/storage/test_impl_sqlalchemy.py b/ceilometer/tests/storage/test_impl_sqlalchemy.py index f8f0ec283..2d72b8e91 100644 --- a/ceilometer/tests/storage/test_impl_sqlalchemy.py +++ b/ceilometer/tests/storage/test_impl_sqlalchemy.py @@ -34,7 +34,6 @@ from ceilometer.storage import models from ceilometer.storage.sqlalchemy import models as sql_models from ceilometer.tests import db as tests_db from ceilometer.tests.storage import test_storage_scenarios as scenarios -from ceilometer import utils class EventTestBase(tests_db.TestBase): @@ -147,7 +146,7 @@ class EventTest(EventTestBase): self.assertIsNone(trait.t_int) self.assertIsNone(trait.t_string) self.assertIsNone(trait.t_float) - self.assertEqual(trait.t_datetime, utils.dt_to_decimal(now)) + self.assertEqual(trait.t_datetime, now) self.assertIsNotNone(trait.trait_type.desc) def test_bad_event(self): diff --git a/ceilometer/tests/storage/test_storage_scenarios.py b/ceilometer/tests/storage/test_storage_scenarios.py index 448baa087..d08ca87d1 100644 --- a/ceilometer/tests/storage/test_storage_scenarios.py +++ b/ceilometer/tests/storage/test_storage_scenarios.py @@ -2145,7 +2145,7 @@ class EventTest(EventTestBase): class GetEventTest(EventTestBase): def prepare_data(self): - event_models = [] + self.event_models = [] base = 0 self.start = datetime.datetime(2013, 12, 31, 5, 0) now = self.start @@ -2160,14 +2160,28 @@ class GetEventTest(EventTestBase): ('trait_C', models.Trait.FLOAT_TYPE, float(base) + 0.123456), ('trait_D', models.Trait.DATETIME_TYPE, now)]] - event_models.append( + self.event_models.append( models.Event("id_%s_%d" % (event_type, base), event_type, now, trait_models)) base += 100 now = now + datetime.timedelta(hours=1) self.end = now - self.conn.record_events(event_models) + self.conn.record_events(self.event_models) + + def test_generated_is_datetime(self): + event_filter = storage.EventFilter(self.start, self.end) + events = self.conn.get_events(event_filter) + self.assertEqual(6, len(events)) + for i, event in enumerate(events): + self.assertTrue(isinstance(event.generated, datetime.datetime)) + self.assertEqual(event.generated, + self.event_models[i].generated) + model_traits = self.event_models[i].traits + for j, trait in enumerate(event.traits): + if trait.dtype == models.Trait.DATETIME_TYPE: + self.assertTrue(isinstance(trait.value, datetime.datetime)) + self.assertEqual(trait.value, model_traits[j].value) def test_simple_get(self): event_filter = storage.EventFilter(self.start, self.end)