openstack/aodh
Code Issues Proposed changes

sqlalchemy: use DATETIME(fsp=6) rather than DECIMAL

Browse Source 
This migrates the old DECIMAL based format to the new DATETIME format available
in recent versions of MySQL.

Change-Id: I5dc7a7c2586feec72a1a2b13865e353a844a1785
This commit is contained in:
Julien Danjou 2016-09-19 17:34:51 +02:00
parent e8eafdbba7
commit d1391f7fa0
5 changed files with 97 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
aodh/storage/sqlalchemy/alembic/versions/12fe8fac9fe4_initial_base.py
View File

@ -29,10 +29,25 @@ depends_on = None
from alembic import op
import sqlalchemy as sa
from sqlalchemy import types
import aodh.storage.sqlalchemy.models
class PreciseTimestamp(types.TypeDecorator):
"""Represents a timestamp precise to the microsecond."""
impl = sa.DateTime
def load_dialect_impl(self, dialect):
if dialect.name == 'mysql':
return dialect.type_descriptor(
types.DECIMAL(precision=20,
scale=6,
asdecimal=True))
return dialect.type_descriptor(self.impl)
def upgrade():
op.create_table(
'alarm_history',
@ -44,7 +59,7 @@ def upgrade():
sa.Column('type', sa.String(length=20), nullable=True),
sa.Column('detail', sa.Text(), nullable=True),
sa.Column('timestamp',
aodh.storage.sqlalchemy.models.PreciseTimestamp(),
PreciseTimestamp(),
nullable=True),
sa.PrimaryKeyConstraint('event_id')
)
@ -60,13 +75,13 @@ def upgrade():
sa.Column('severity', sa.String(length=50), nullable=True),
sa.Column('description', sa.Text(), nullable=True),
sa.Column('timestamp',
aodh.storage.sqlalchemy.models.PreciseTimestamp(),
PreciseTimestamp(),
nullable=True),
sa.Column('user_id', sa.String(length=128), nullable=True),
sa.Column('project_id', sa.String(length=128), nullable=True),
sa.Column('state', sa.String(length=255), nullable=True),
sa.Column('state_timestamp',
aodh.storage.sqlalchemy.models.PreciseTimestamp(),
PreciseTimestamp(),
nullable=True),
sa.Column('ok_actions',
aodh.storage.sqlalchemy.models.JSONEncodedDict(),

68
aodh/storage/sqlalchemy/alembic/versions/367aadf5485f_precisetimestamp_to_datetime.py Normal file
View File

@ -0,0 +1,68 @@
# -*- encoding: utf-8 -*-
#
# Copyright 2016 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
"""precisetimestamp_to_datetime
Revision ID: 367aadf5485f
Revises: f8c31b1ffe11
Create Date: 2016-09-19 16:43:34.379029
"""
# revision identifiers, used by Alembic.
revision = '367aadf5485f'
down_revision = 'f8c31b1ffe11'
branch_labels = None
depends_on = None
from alembic import op
import sqlalchemy as sa
from sqlalchemy import func
from aodh.storage.sqlalchemy import models
def upgrade():
bind = op.get_bind()
if bind and bind.engine.name == "mysql":
# NOTE(jd) So that crappy engine that is MySQL does not have "ALTER
# TABLE … USING …". We need to copy everything and convert…
for table_name, column_name in (("alarm", "timestamp"),
("alarm", "state_timestamp"),
("alarm_change", "timestamp")):
existing_type = sa.types.DECIMAL(
precision=20, scale=6, asdecimal=True)
existing_col = sa.Column(
column_name,
existing_type,
nullable=True)
temp_col = sa.Column(
column_name + "_ts",
models.TimestampUTC(),
nullable=True)
op.add_column(table_name, temp_col)
t = sa.sql.table(table_name, existing_col, temp_col)
op.execute(t.update().values(
**{column_name + "_ts": func.from_unixtime(existing_col)}))
op.drop_column(table_name, column_name)
op.alter_column(table_name,
column_name + "_ts",
nullable=True,
type_=models.TimestampUTC(),
existing_nullable=True,
existing_type=existing_type,
new_column_name=column_name)

51
aodh/storage/sqlalchemy/models.py
View File

@ -13,16 +13,12 @@
"""
SQLAlchemy models for aodh data.
"""
import calendar
import datetime
import decimal
import json
from oslo_utils import timeutils
from oslo_utils import units
import six
from sqlalchemy import Column, String, Index, Boolean, Text, DateTime
from sqlalchemy.dialects.mysql import DECIMAL
from sqlalchemy.dialects import mysql
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.types import TypeDecorator
@ -45,48 +41,15 @@ class JSONEncodedDict(TypeDecorator):
return value
class PreciseTimestamp(TypeDecorator):
class TimestampUTC(TypeDecorator):
"""Represents a timestamp precise to the microsecond."""
impl = DateTime
def load_dialect_impl(self, dialect):
if dialect.name == 'mysql':
return dialect.type_descriptor(DECIMAL(precision=20,
scale=6,
asdecimal=True))
return dialect.type_descriptor(self.impl)
@staticmethod
def process_bind_param(value, dialect):
if value is None:
return value
elif dialect.name == 'mysql':
decimal.getcontext().prec = 30
return (
decimal.Decimal(
str(calendar.timegm(value.utctimetuple()))) +
(decimal.Decimal(str(value.microsecond)) /
decimal.Decimal("1000000.0"))
)
return value
def compare_against_backend(self, dialect, conn_type):
if dialect.name == 'mysql':
return issubclass(type(conn_type), DECIMAL)
return issubclass(type(conn_type), DateTime)
@staticmethod
def process_result_value(value, dialect):
if value is None:
return value
elif dialect.name == 'mysql':
integer = int(value)
micro = (value
- decimal.Decimal(integer)) * decimal.Decimal(units.M)
daittyme = datetime.datetime.utcfromtimestamp(integer)
return daittyme.replace(microsecond=int(round(micro)))
return value
return dialect.type_descriptor(mysql.DATETIME(fsp=6))
return self.impl
class AodhBase(object):
@ -125,13 +88,13 @@ class Alarm(Base):
type = Column(String(50))
severity = Column(String(50))
description = Column(Text)
timestamp = Column(PreciseTimestamp, default=lambda: timeutils.utcnow())
timestamp = Column(TimestampUTC, default=lambda: timeutils.utcnow())
user_id = Column(String(128))
project_id = Column(String(128))
state = Column(String(255))
state_timestamp = Column(PreciseTimestamp,
state_timestamp = Column(TimestampUTC,
default=lambda: timeutils.utcnow())
ok_actions = Column(JSONEncodedDict)
@ -156,5 +119,5 @@ class AlarmChange(Base):
user_id = Column(String(128))
type = Column(String(20))
detail = Column(Text)
timestamp = Column(PreciseTimestamp, default=lambda: timeutils.utcnow())
timestamp = Column(TimestampUTC, default=lambda: timeutils.utcnow())
severity = Column(String(50))

86
aodh/tests/functional/storage/sqlalchemy/test_models.py
View File

@ -1,86 +0,0 @@
#
# Copyright 2013 Rackspace Hosting
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
import mock
from oslotest import base
import sqlalchemy
from sqlalchemy.dialects.mysql import DECIMAL
from sqlalchemy.types import NUMERIC
from aodh.storage.sqlalchemy import models
class PreciseTimestampTest(base.BaseTestCase):
@staticmethod
def fake_dialect(name):
def _type_descriptor_mock(desc):
if type(desc) == DECIMAL:
return NUMERIC(precision=desc.precision, scale=desc.scale)
else:
return desc
dialect = mock.MagicMock()
dialect.name = name
dialect.type_descriptor = _type_descriptor_mock
return dialect
def setUp(self):
super(PreciseTimestampTest, self).setUp()
self._mysql_dialect = self.fake_dialect('mysql')
self._postgres_dialect = self.fake_dialect('postgres')
self._type = models.PreciseTimestamp()
self._date = datetime.datetime(2012, 7, 2, 10, 44)
def test_load_dialect_impl_mysql(self):
result = self._type.load_dialect_impl(self._mysql_dialect)
self.assertEqual(NUMERIC, type(result))
self.assertEqual(20, result.precision)
self.assertEqual(6, result.scale)
self.assertTrue(result.asdecimal)
def test_load_dialect_impl_postgres(self):
result = self._type.load_dialect_impl(self._postgres_dialect)
self.assertEqual(sqlalchemy.DateTime, type(result))
def test_process_bind_param_store_datetime_postgres(self):
result = self._type.process_bind_param(self._date,
self._postgres_dialect)
self.assertEqual(self._date, result)
def test_process_bind_param_store_none_mysql(self):
result = self._type.process_bind_param(None, self._mysql_dialect)
self.assertIsNone(result)
def test_process_bind_param_store_none_postgres(self):
result = self._type.process_bind_param(None,
self._postgres_dialect)
self.assertIsNone(result)
def test_process_result_value_datetime_postgres(self):
result = self._type.process_result_value(self._date,
self._postgres_dialect)
self.assertEqual(self._date, result)
def test_process_result_value_none_mysql(self):
result = self._type.process_result_value(None,
self._mysql_dialect)
self.assertIsNone(result)
def test_process_result_value_none_postgres(self):
result = self._type.process_result_value(None,
self._postgres_dialect)
self.assertIsNone(result)

4
releasenotes/notes/mysql-precise-datetime-e374c77e6707985e.yaml Normal file
View File

@ -0,0 +1,4 @@
---
other:
- Aodh now leverages microseconds timestamps available since MySQL 5.6.4,
meaning it is now the minimum required version of MySQL.