sqlalchemy: use DATETIME(fsp=6) rather than DECIMAL

This migrates the old DECIMAL based format to the new DATETIME format available
in recent versions of MySQL.

Change-Id: I5dc7a7c2586feec72a1a2b13865e353a844a1785

aodh/storage/sqlalchemy/alembic/versions/12fe8fac9fe4_initial_base.py

@@ -29,10 +29,25 @@ depends_on = None
 
 from alembic import op
 import sqlalchemy as sa
+from sqlalchemy import types
 
 import aodh.storage.sqlalchemy.models
 
 
+class PreciseTimestamp(types.TypeDecorator):
+    """Represents a timestamp precise to the microsecond."""
+
+    impl = sa.DateTime
+
+    def load_dialect_impl(self, dialect):
+        if dialect.name == 'mysql':
+            return dialect.type_descriptor(
+                types.DECIMAL(precision=20,
+                              scale=6,
+                              asdecimal=True))
+        return dialect.type_descriptor(self.impl)
+
+
 def upgrade():
     op.create_table(
         'alarm_history',
@@ -44,7 +59,7 @@ def upgrade():
         sa.Column('type', sa.String(length=20), nullable=True),
         sa.Column('detail', sa.Text(), nullable=True),
         sa.Column('timestamp',
-                  aodh.storage.sqlalchemy.models.PreciseTimestamp(),
+                  PreciseTimestamp(),
                   nullable=True),
         sa.PrimaryKeyConstraint('event_id')
     )
@@ -60,13 +75,13 @@ def upgrade():
         sa.Column('severity', sa.String(length=50), nullable=True),
         sa.Column('description', sa.Text(), nullable=True),
         sa.Column('timestamp',
-                  aodh.storage.sqlalchemy.models.PreciseTimestamp(),
+                  PreciseTimestamp(),
                   nullable=True),
         sa.Column('user_id', sa.String(length=128), nullable=True),
         sa.Column('project_id', sa.String(length=128), nullable=True),
         sa.Column('state', sa.String(length=255), nullable=True),
         sa.Column('state_timestamp',
-                  aodh.storage.sqlalchemy.models.PreciseTimestamp(),
+                  PreciseTimestamp(),
                   nullable=True),
         sa.Column('ok_actions',
                   aodh.storage.sqlalchemy.models.JSONEncodedDict(),

aodh/storage/sqlalchemy/alembic/versions/367aadf5485f_precisetimestamp_to_datetime.py

@@ -0,0 +1,68 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright 2016 OpenStack Foundation
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+"""precisetimestamp_to_datetime
+
+Revision ID: 367aadf5485f
+Revises: f8c31b1ffe11
+Create Date: 2016-09-19 16:43:34.379029
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '367aadf5485f'
+down_revision = 'f8c31b1ffe11'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy import func
+
+from aodh.storage.sqlalchemy import models
+
+
+def upgrade():
+    bind = op.get_bind()
+    if bind and bind.engine.name == "mysql":
+        # NOTE(jd) So that crappy engine that is MySQL does not have "ALTER
+        # TABLE … USING …". We need to copy everything and convert…
+        for table_name, column_name in (("alarm", "timestamp"),
+                                        ("alarm", "state_timestamp"),
+                                        ("alarm_change", "timestamp")):
+            existing_type = sa.types.DECIMAL(
+                precision=20, scale=6, asdecimal=True)
+            existing_col = sa.Column(
+                column_name,
+                existing_type,
+                nullable=True)
+            temp_col = sa.Column(
+                column_name + "_ts",
+                models.TimestampUTC(),
+                nullable=True)
+            op.add_column(table_name, temp_col)
+            t = sa.sql.table(table_name, existing_col, temp_col)
+            op.execute(t.update().values(
+                **{column_name + "_ts": func.from_unixtime(existing_col)}))
+            op.drop_column(table_name, column_name)
+            op.alter_column(table_name,
+                            column_name + "_ts",
+                            nullable=True,
+                            type_=models.TimestampUTC(),
+                            existing_nullable=True,
+                            existing_type=existing_type,
+                            new_column_name=column_name)

aodh/storage/sqlalchemy/models.py

@@ -13,16 +13,12 @@
 """
 SQLAlchemy models for aodh data.
 """
-import calendar
-import datetime
-import decimal
 import json
 
 from oslo_utils import timeutils
-from oslo_utils import units
 import six
 from sqlalchemy import Column, String, Index, Boolean, Text, DateTime
-from sqlalchemy.dialects.mysql import DECIMAL
+from sqlalchemy.dialects import mysql
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.types import TypeDecorator
 
@@ -45,48 +41,15 @@ class JSONEncodedDict(TypeDecorator):
         return value
 
 
-class PreciseTimestamp(TypeDecorator):
+class TimestampUTC(TypeDecorator):
     """Represents a timestamp precise to the microsecond."""
 
     impl = DateTime
 
     def load_dialect_impl(self, dialect):
         if dialect.name == 'mysql':
-            return dialect.type_descriptor(DECIMAL(precision=20,
+            return dialect.type_descriptor(mysql.DATETIME(fsp=6))
-                                                   scale=6,
+        return self.impl
-                                                   asdecimal=True))
-        return dialect.type_descriptor(self.impl)
-
-    @staticmethod
-    def process_bind_param(value, dialect):
-        if value is None:
-            return value
-        elif dialect.name == 'mysql':
-            decimal.getcontext().prec = 30
-            return (
-                decimal.Decimal(
-                    str(calendar.timegm(value.utctimetuple()))) +
-                (decimal.Decimal(str(value.microsecond)) /
-                 decimal.Decimal("1000000.0"))
-            )
-        return value
-
-    def compare_against_backend(self, dialect, conn_type):
-        if dialect.name == 'mysql':
-            return issubclass(type(conn_type), DECIMAL)
-        return issubclass(type(conn_type), DateTime)
-
-    @staticmethod
-    def process_result_value(value, dialect):
-        if value is None:
-            return value
-        elif dialect.name == 'mysql':
-            integer = int(value)
-            micro = (value
-                     - decimal.Decimal(integer)) * decimal.Decimal(units.M)
-            daittyme = datetime.datetime.utcfromtimestamp(integer)
-            return daittyme.replace(microsecond=int(round(micro)))
-        return value
 
 
 class AodhBase(object):
@@ -125,13 +88,13 @@ class Alarm(Base):
     type = Column(String(50))
     severity = Column(String(50))
     description = Column(Text)
-    timestamp = Column(PreciseTimestamp, default=lambda: timeutils.utcnow())
+    timestamp = Column(TimestampUTC, default=lambda: timeutils.utcnow())
 
     user_id = Column(String(128))
     project_id = Column(String(128))
 
     state = Column(String(255))
-    state_timestamp = Column(PreciseTimestamp,
+    state_timestamp = Column(TimestampUTC,
                              default=lambda: timeutils.utcnow())
 
     ok_actions = Column(JSONEncodedDict)
@@ -156,5 +119,5 @@ class AlarmChange(Base):
     user_id = Column(String(128))
     type = Column(String(20))
     detail = Column(Text)
-    timestamp = Column(PreciseTimestamp, default=lambda: timeutils.utcnow())
+    timestamp = Column(TimestampUTC, default=lambda: timeutils.utcnow())
     severity = Column(String(50))

aodh/tests/functional/storage/sqlalchemy/test_models.py

@@ -1,86 +0,0 @@
-#
-# Copyright 2013 Rackspace Hosting
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import datetime
-
-import mock
-from oslotest import base
-import sqlalchemy
-from sqlalchemy.dialects.mysql import DECIMAL
-from sqlalchemy.types import NUMERIC
-
-from aodh.storage.sqlalchemy import models
-
-
-class PreciseTimestampTest(base.BaseTestCase):
-
-    @staticmethod
-    def fake_dialect(name):
-        def _type_descriptor_mock(desc):
-            if type(desc) == DECIMAL:
-                return NUMERIC(precision=desc.precision, scale=desc.scale)
-            else:
-                return desc
-        dialect = mock.MagicMock()
-        dialect.name = name
-        dialect.type_descriptor = _type_descriptor_mock
-        return dialect
-
-    def setUp(self):
-        super(PreciseTimestampTest, self).setUp()
-        self._mysql_dialect = self.fake_dialect('mysql')
-        self._postgres_dialect = self.fake_dialect('postgres')
-        self._type = models.PreciseTimestamp()
-        self._date = datetime.datetime(2012, 7, 2, 10, 44)
-
-    def test_load_dialect_impl_mysql(self):
-        result = self._type.load_dialect_impl(self._mysql_dialect)
-        self.assertEqual(NUMERIC, type(result))
-        self.assertEqual(20, result.precision)
-        self.assertEqual(6, result.scale)
-        self.assertTrue(result.asdecimal)
-
-    def test_load_dialect_impl_postgres(self):
-        result = self._type.load_dialect_impl(self._postgres_dialect)
-        self.assertEqual(sqlalchemy.DateTime, type(result))
-
-    def test_process_bind_param_store_datetime_postgres(self):
-        result = self._type.process_bind_param(self._date,
-                                               self._postgres_dialect)
-        self.assertEqual(self._date, result)
-
-    def test_process_bind_param_store_none_mysql(self):
-        result = self._type.process_bind_param(None, self._mysql_dialect)
-        self.assertIsNone(result)
-
-    def test_process_bind_param_store_none_postgres(self):
-        result = self._type.process_bind_param(None,
-                                               self._postgres_dialect)
-        self.assertIsNone(result)
-
-    def test_process_result_value_datetime_postgres(self):
-        result = self._type.process_result_value(self._date,
-                                                 self._postgres_dialect)
-        self.assertEqual(self._date, result)
-
-    def test_process_result_value_none_mysql(self):
-        result = self._type.process_result_value(None,
-                                                 self._mysql_dialect)
-        self.assertIsNone(result)
-
-    def test_process_result_value_none_postgres(self):
-        result = self._type.process_result_value(None,
-                                                 self._postgres_dialect)
-        self.assertIsNone(result)

releasenotes/notes/mysql-precise-datetime-e374c77e6707985e.yaml

@@ -0,0 +1,4 @@
+---
+other:
+  - Aodh now leverages microseconds timestamps available since MySQL 5.6.4,
+    meaning it is now the minimum required version of MySQL.