Merge "Add alembic support for aodh"

aodh/storage/impl_sqlalchemy.py

@@ -14,7 +14,11 @@
 
 from __future__ import absolute_import
 import datetime
+import os.path
 
+from alembic import command
+from alembic import config
+from alembic import migration
 from oslo_db.sqlalchemy import session as db_session
 from oslo_log import log
 from oslo_utils import timeutils
@@ -58,11 +62,33 @@ class Connection(base.Connection):
         # in storage.__init__.get_connection_from_config function
         options = dict(conf.database.items())
         options['max_retries'] = 0
+        self.conf = conf
         self._engine_facade = db_session.EngineFacade(url, **options)
 
-    def upgrade(self):
-        engine = self._engine_facade.get_engine()
-        models.Base.metadata.create_all(engine)
+    def disconnect(self):
+        self._engine_facade.get_engine().dispose()
+
+    def _get_alembic_config(self):
+        cfg = config.Config(
+            "%s/sqlalchemy/alembic/alembic.ini" % os.path.dirname(__file__))
+        cfg.set_main_option('sqlalchemy.url',
+                            self.conf.database.connection)
+        return cfg
+
+    def upgrade(self, nocreate=False):
+        cfg = self._get_alembic_config()
+        cfg.conf = self.conf
+        if nocreate:
+            command.upgrade(cfg, "head")
+        else:
+            engine = self._engine_facade.get_engine()
+            ctxt = migration.MigrationContext.configure(engine.connect())
+            current_version = ctxt.get_current_revision()
+            if current_version is None:
+                models.Base.metadata.create_all(engine)
+                command.stamp(cfg, "head")
+            else:
+                command.upgrade(cfg, "head")
 
     def clear(self):
         engine = self._engine_facade.get_engine()

aodh/storage/sqlalchemy/alembic/alembic.ini

@@ -0,0 +1,37 @@
+[alembic]
+script_location = aodh.storage.sqlalchemy:alembic
+sqlalchemy.url =
+
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = WARN
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

aodh/storage/sqlalchemy/alembic/env.py

@@ -0,0 +1,92 @@
+#
+# Copyright 2015 Huawei Technologies Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from __future__ import with_statement
+from alembic import context
+from logging.config import fileConfig
+
+from aodh.storage import impl_sqlalchemy
+from aodh.storage.sqlalchemy import models
+
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = models.Base.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    conf = config.conf
+    context.configure(url=conf.database.connection,
+                      target_metadata=target_metadata)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    conf = config.conf
+    conn = impl_sqlalchemy.Connection(conf, conf.database.connection)
+    connectable = conn._engine_facade.get_engine()
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+    conn.disconnect()
+
+if not hasattr(config, "conf"):
+    from aodh import service
+    config.conf = service.prepare_service([])
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

aodh/storage/sqlalchemy/alembic/script.py.mako

@@ -0,0 +1,39 @@
+# Copyright ${create_date.year} OpenStack Foundation
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

aodh/storage/sqlalchemy/alembic/versions/12fe8fac9fe4_initial_base.py

@@ -0,0 +1,92 @@
+# Copyright 2015 OpenStack Foundation
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+"""initial base
+
+Revision ID: 12fe8fac9fe4
+Revises:
+Create Date: 2015-07-28 17:38:37.022899
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '12fe8fac9fe4'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+
+import aodh.storage.sqlalchemy.models
+
+
+def upgrade():
+    op.create_table(
+        'alarm_history',
+        sa.Column('event_id', sa.String(length=128), nullable=False),
+        sa.Column('alarm_id', sa.String(length=128), nullable=True),
+        sa.Column('on_behalf_of', sa.String(length=128), nullable=True),
+        sa.Column('project_id', sa.String(length=128), nullable=True),
+        sa.Column('user_id', sa.String(length=128), nullable=True),
+        sa.Column('type', sa.String(length=20), nullable=True),
+        sa.Column('detail', sa.Text(), nullable=True),
+        sa.Column('timestamp',
+                  aodh.storage.sqlalchemy.models.PreciseTimestamp(),
+                  nullable=True),
+        sa.PrimaryKeyConstraint('event_id')
+    )
+    op.create_index(
+        'ix_alarm_history_alarm_id', 'alarm_history', ['alarm_id'],
+        unique=False)
+    op.create_table(
+        'alarm',
+        sa.Column('alarm_id', sa.String(length=128), nullable=False),
+        sa.Column('enabled', sa.Boolean(), nullable=True),
+        sa.Column('name', sa.Text(), nullable=True),
+        sa.Column('type', sa.String(length=50), nullable=True),
+        sa.Column('severity', sa.String(length=50), nullable=True),
+        sa.Column('description', sa.Text(), nullable=True),
+        sa.Column('timestamp',
+                  aodh.storage.sqlalchemy.models.PreciseTimestamp(),
+                  nullable=True),
+        sa.Column('user_id', sa.String(length=128), nullable=True),
+        sa.Column('project_id', sa.String(length=128), nullable=True),
+        sa.Column('state', sa.String(length=255), nullable=True),
+        sa.Column('state_timestamp',
+                  aodh.storage.sqlalchemy.models.PreciseTimestamp(),
+                  nullable=True),
+        sa.Column('ok_actions',
+                  aodh.storage.sqlalchemy.models.JSONEncodedDict(),
+                  nullable=True),
+        sa.Column('alarm_actions',
+                  aodh.storage.sqlalchemy.models.JSONEncodedDict(),
+                  nullable=True),
+        sa.Column('insufficient_data_actions',
+                  aodh.storage.sqlalchemy.models.JSONEncodedDict(),
+                  nullable=True),
+        sa.Column('repeat_actions', sa.Boolean(), nullable=True),
+        sa.Column('rule',
+                  aodh.storage.sqlalchemy.models.JSONEncodedDict(),
+                  nullable=True),
+        sa.Column('time_constraints',
+                  aodh.storage.sqlalchemy.models.JSONEncodedDict(),
+                  nullable=True),
+        sa.PrimaryKeyConstraint('alarm_id')
+    )
+    op.create_index(
+        'ix_alarm_project_id', 'alarm', ['project_id'], unique=False)
+    op.create_index(
+        'ix_alarm_user_id', 'alarm', ['user_id'], unique=False)

aodh/storage/sqlalchemy/models.py

@@ -53,7 +53,7 @@ class PreciseTimestamp(TypeDecorator):
             return dialect.type_descriptor(DECIMAL(precision=20,
                                                    scale=6,
                                                    asdecimal=True))
-        return self.impl
+        return dialect.type_descriptor(self.impl)
 
     @staticmethod
     def process_bind_param(value, dialect):
@@ -63,6 +63,11 @@ class PreciseTimestamp(TypeDecorator):
             return utils.dt_to_decimal(value)
         return value
 
+    def compare_against_backend(self, dialect, conn_type):
+        if dialect.name == 'mysql':
+            return issubclass(type(conn_type), DECIMAL)
+        return issubclass(type(conn_type), type(self.impl))
+
     @staticmethod
     def process_result_value(value, dialect):
         if value is None:

aodh/tests/storage/sqlalchemy/test_migrations.py

@@ -0,0 +1,62 @@
+#
+# Copyright 2015 Huawei Technologies Co., Ltd.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+import abc
+
+import mock
+from oslo_config import fixture as fixture_config
+from oslo_db.sqlalchemy import test_migrations
+import six
+import six.moves.urllib.parse as urlparse
+
+from aodh import service
+from aodh.storage import impl_sqlalchemy
+from aodh.storage.sqlalchemy import models
+from aodh.tests import base
+
+
+class ABCSkip(base.SkipNotImplementedMeta, abc.ABCMeta):
+    pass
+
+
+class ModelsMigrationsSync(
+        six.with_metaclass(ABCSkip,
+                           base.BaseTestCase,
+                           test_migrations.ModelsMigrationsSync)):
+
+    def setUp(self):
+        super(ModelsMigrationsSync, self).setUp()
+        self.db = mock.Mock()
+        conf = service.prepare_service([])
+        self.conf = self.useFixture(fixture_config.Config(conf)).conf
+        db_url = self.conf.database.connection
+        if not db_url:
+            self.skipTest("The db connection option should be specified.")
+        connection_scheme = urlparse.urlparse(db_url).scheme
+        engine_name = connection_scheme.split('+')[0]
+        if engine_name not in ('postgresql', 'mysql', 'sqlite'):
+            self.skipTest("This test only works with PostgreSQL or MySQL or"
+                          " SQLite")
+        self.conn = impl_sqlalchemy.Connection(self.conf,
+                                               self.conf.database.connection)
+
+    @staticmethod
+    def get_metadata():
+        return models.Base.metadata
+
+    def get_engine(self):
+        return self.conn._engine_facade.get_engine()
+
+    def db_sync(self, engine):
+        self.conn.upgrade(nocreate=True)

aodh/tests/storage/sqlalchemy/test_models.py

@@ -32,6 +32,8 @@ class PreciseTimestampTest(base.BaseTestCase):
         def _type_descriptor_mock(desc):
             if type(desc) == DECIMAL:
                 return NUMERIC(precision=desc.precision, scale=desc.scale)
+            else:
+                return desc
         dialect = mock.MagicMock()
         dialect.name = name
         dialect.type_descriptor = _type_descriptor_mock

doc/source/conf.py

@@ -85,7 +85,7 @@ copyright = u'2012-2015, OpenStack Foundation'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-exclude_patterns = ['**/#*', '**~', '**/#*#']
+exclude_patterns = ['**/#*', '**~', '**/#*#', '**/*alembic*']
 
 # The reST default role (used for this markup: `text`)
 # to use for all documents.

requirements.txt

@@ -2,6 +2,7 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 
+alembic>=0.7.2
 retrying!=1.3.0,>=1.2.3 # Apache-2.0
 croniter>=0.3.4 # MIT License
 eventlet>=0.17.4

setup.cfg

@@ -75,6 +75,7 @@ source-dir = doc/source
 [pbr]
 warnerrors = true
 autodoc_index_modules = true
+autodoc_exclude_modules = aodh.storage.sqlalchemy.alembic.*
 
 [extract_messages]
 keywords = _ gettext ngettext l_ lazy_gettext