Merge "Convert event timestamp to PrecisionTimestamp"

ceilometer/storage/impl_sqlalchemy.py

@@ -867,8 +867,6 @@ class Connection(base.Connection):
         values = {'t_string': None, 't_float': None,
                   't_int': None, 't_datetime': None}
         value = trait_model.value
-        if trait_model.dtype == api_models.Trait.DATETIME_TYPE:
-            value = utils.dt_to_decimal(value)
         values[value_map[trait_model.dtype]] = value
         return models.Trait(trait_type, event, **values)
 
@@ -897,8 +895,8 @@ class Connection(base.Connection):
             event_type = cls._get_or_create_event_type(event_model.event_type,
                                                        session=session)
 
-            generated = utils.dt_to_decimal(event_model.generated)
+            event = models.Event(event_model.message_id, event_type,
-            event = models.Event(event_model.message_id, event_type, generated)
+                                 event_model.generated)
             session.add(event)
 
             new_traits = []
@@ -948,8 +946,8 @@ class Connection(base.Connection):
         :param event_filter: EventFilter instance
         """
 
-        start = utils.dt_to_decimal(event_filter.start_time)
+        start = event_filter.start_time
-        end = utils.dt_to_decimal(event_filter.end_time)
+        end = event_filter.end_time
         session = sqlalchemy_session.get_session()
         with session.begin():
             event_query = session.query(models.Event)
@@ -996,8 +994,7 @@ class Connection(base.Connection):
                         elif key == 't_int':
                             conditions.append(models.Trait.t_int == value)
                         elif key == 't_datetime':
-                            dt = utils.dt_to_decimal(value)
+                            conditions.append(models.Trait.t_datetime == value)
-                            conditions.append(models.Trait.t_datetime == dt)
                         elif key == 't_float':
                             conditions.append(models.Trait.t_float == value)
 
@@ -1034,11 +1031,10 @@ class Connection(base.Connection):
             for trait in query.all():
                 event = event_models_dict.get(trait.event_id)
                 if not event:
-                    generated = utils.decimal_to_dt(trait.event.generated)
                     event = api_models.Event(
                         trait.event.message_id,
                         trait.event.event_type.desc,
-                        generated, [])
+                        trait.event.generated, [])
                     event_models_dict[trait.event_id] = event
                 trait_model = api_models.Trait(trait.trait_type.desc,
                                                trait.trait_type.data_type,
@@ -1115,7 +1111,7 @@ class Connection(base.Connection):
                      .join(models.EventType,
                            and_(models.EventType.id ==
                                 models.Event.event_type_id,
-                           models.EventType.desc == event_type)))
+                                models.EventType.desc == event_type)))
 
             for trait in query.all():
                 type = trait.trait_type

ceilometer/storage/sqlalchemy/migrate_repo/versions/024_event_use_floatingprecision.py

@@ -0,0 +1,81 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright © 2013 eNovance SAS <licensing@enovance.com>
+#
+# Author: Mehdi Abaakouk <mehdi.abaakouk@enovance.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import sqlalchemy as sa
+
+from ceilometer.storage.sqlalchemy import models
+
+
+def _paged(query, size):
+    offset = 0
+    while True:
+        page = query.offset(offset).limit(size).execute()
+        if page.rowcount <= 0:
+            # There are no more rows
+            break
+        for row in page:
+            yield row
+        offset += size
+
+
+def _convert_data_type(table, col, from_t, to_t, pk_attr='id', index=False):
+    temp_col_n = 'convert_data_type_temp_col'
+    # Override column we're going to convert with from_t, since the type we're
+    # replacing could be custom and we need to tell SQLALchemy how to perform
+    # CRUD operations with it.
+    table = sa.Table(table.name, table.metadata, sa.Column(col, from_t),
+                     extend_existing=True)
+    sa.Column(temp_col_n, to_t).create(table)
+
+    key_attr = getattr(table.c, pk_attr)
+    orig_col = getattr(table.c, col)
+    new_col = getattr(table.c, temp_col_n)
+
+    query = sa.select([key_attr, orig_col])
+    for key, value in _paged(query, 1000):
+        table.update().where(key_attr == key)\
+            .values({temp_col_n: value}).execute()
+
+    orig_col.drop()
+    new_col.alter(name=col)
+    if index:
+        sa.Index('ix_%s_%s' % (table.name, col), new_col).create()
+
+
+def upgrade(migrate_engine):
+    if migrate_engine.name == 'mysql':
+        meta = sa.MetaData(bind=migrate_engine)
+        event = sa.Table('event', meta, autoload=True)
+        _convert_data_type(event, 'generated', sa.Float(),
+                           models.PreciseTimestamp(),
+                           pk_attr='id', index=True)
+        trait = sa.Table('trait', meta, autoload=True)
+        _convert_data_type(trait, 't_datetime', sa.Float(),
+                           models.PreciseTimestamp(),
+                           pk_attr='id', index=True)
+
+
+def downgrade(migrate_engine):
+    if migrate_engine.name == 'mysql':
+        meta = sa.MetaData(bind=migrate_engine)
+        event = sa.Table('event', meta, autoload=True)
+        _convert_data_type(event, 'generated', models.PreciseTimestamp(),
+                           sa.Float(), pk_attr='id', index=True)
+        trait = sa.Table('trait', meta, autoload=True)
+        _convert_data_type(trait, 't_datetime', models.PreciseTimestamp(),
+                           sa.Float(), pk_attr='id', index=True)

ceilometer/storage/sqlalchemy/models.py

@@ -320,7 +320,7 @@ class Event(Base):
     )
     id = Column(Integer, primary_key=True)
     message_id = Column(String(50), unique=True)
-    generated = Column(Float(asdecimal=True))
+    generated = Column(PreciseTimestamp())
 
     event_type_id = Column(Integer, ForeignKey('event_type.id'))
     event_type = relationship("EventType", backref=backref('event_type'))
@@ -379,7 +379,7 @@ class Trait(Base):
     t_string = Column(String(255), nullable=True, default=None)
     t_float = Column(Float, nullable=True, default=None)
     t_int = Column(Integer, nullable=True, default=None)
-    t_datetime = Column(Float(asdecimal=True), nullable=True, default=None)
+    t_datetime = Column(PreciseTimestamp(), nullable=True, default=None)
 
     event_id = Column(Integer, ForeignKey('event.id'))
     event = relationship("Event", backref=backref('event', order_by=id))
@@ -409,7 +409,7 @@ class Trait(Base):
         if dtype == api_models.Trait.FLOAT_TYPE:
             return self.t_float
         if dtype == api_models.Trait.DATETIME_TYPE:
-            return utils.decimal_to_dt(self.t_datetime)
+            return self.t_datetime
         if dtype == api_models.Trait.TEXT_TYPE:
             return self.t_string
 

ceilometer/tests/storage/test_impl_sqlalchemy.py

@@ -34,7 +34,6 @@ from ceilometer.storage import models
 from ceilometer.storage.sqlalchemy import models as sql_models
 from ceilometer.tests import db as tests_db
 from ceilometer.tests.storage import test_storage_scenarios as scenarios
-from ceilometer import utils
 
 
 class EventTestBase(tests_db.TestBase):
@@ -147,7 +146,7 @@ class EventTest(EventTestBase):
         self.assertIsNone(trait.t_int)
         self.assertIsNone(trait.t_string)
         self.assertIsNone(trait.t_float)
-        self.assertEqual(trait.t_datetime, utils.dt_to_decimal(now))
+        self.assertEqual(trait.t_datetime, now)
         self.assertIsNotNone(trait.trait_type.desc)
 
     def test_bad_event(self):

ceilometer/tests/storage/test_storage_scenarios.py

@@ -2145,7 +2145,7 @@ class EventTest(EventTestBase):
 
 class GetEventTest(EventTestBase):
     def prepare_data(self):
-        event_models = []
+        self.event_models = []
         base = 0
         self.start = datetime.datetime(2013, 12, 31, 5, 0)
         now = self.start
@@ -2160,14 +2160,28 @@ class GetEventTest(EventTestBase):
                         ('trait_C', models.Trait.FLOAT_TYPE,
                             float(base) + 0.123456),
                         ('trait_D', models.Trait.DATETIME_TYPE, now)]]
-            event_models.append(
+            self.event_models.append(
                 models.Event("id_%s_%d" % (event_type, base),
                              event_type, now, trait_models))
             base += 100
             now = now + datetime.timedelta(hours=1)
         self.end = now
 
-        self.conn.record_events(event_models)
+        self.conn.record_events(self.event_models)
+
+    def test_generated_is_datetime(self):
+        event_filter = storage.EventFilter(self.start, self.end)
+        events = self.conn.get_events(event_filter)
+        self.assertEqual(6, len(events))
+        for i, event in enumerate(events):
+            self.assertTrue(isinstance(event.generated, datetime.datetime))
+            self.assertEqual(event.generated,
+                             self.event_models[i].generated)
+            model_traits = self.event_models[i].traits
+            for j, trait in enumerate(event.traits):
+                if trait.dtype == models.Trait.DATETIME_TYPE:
+                    self.assertTrue(isinstance(trait.value, datetime.datetime))
+                    self.assertEqual(trait.value, model_traits[j].value)
 
     def test_simple_get(self):
         event_filter = storage.EventFilter(self.start, self.end)