diff --git a/aodh/storage/impl_sqlalchemy.py b/aodh/storage/impl_sqlalchemy.py index d045e5a81..621886c24 100644 --- a/aodh/storage/impl_sqlalchemy.py +++ b/aodh/storage/impl_sqlalchemy.py @@ -14,7 +14,11 @@ from __future__ import absolute_import import datetime +import os.path +from alembic import command +from alembic import config +from alembic import migration from oslo_db.sqlalchemy import session as db_session from oslo_log import log from oslo_utils import timeutils @@ -58,11 +62,33 @@ class Connection(base.Connection): # in storage.__init__.get_connection_from_config function options = dict(conf.database.items()) options['max_retries'] = 0 + self.conf = conf self._engine_facade = db_session.EngineFacade(url, **options) - def upgrade(self): - engine = self._engine_facade.get_engine() - models.Base.metadata.create_all(engine) + def disconnect(self): + self._engine_facade.get_engine().dispose() + + def _get_alembic_config(self): + cfg = config.Config( + "%s/sqlalchemy/alembic/alembic.ini" % os.path.dirname(__file__)) + cfg.set_main_option('sqlalchemy.url', + self.conf.database.connection) + return cfg + + def upgrade(self, nocreate=False): + cfg = self._get_alembic_config() + cfg.conf = self.conf + if nocreate: + command.upgrade(cfg, "head") + else: + engine = self._engine_facade.get_engine() + ctxt = migration.MigrationContext.configure(engine.connect()) + current_version = ctxt.get_current_revision() + if current_version is None: + models.Base.metadata.create_all(engine) + command.stamp(cfg, "head") + else: + command.upgrade(cfg, "head") def clear(self): engine = self._engine_facade.get_engine() diff --git a/aodh/storage/sqlalchemy/alembic/alembic.ini b/aodh/storage/sqlalchemy/alembic/alembic.ini new file mode 100644 index 000000000..57732fed3 --- /dev/null +++ b/aodh/storage/sqlalchemy/alembic/alembic.ini @@ -0,0 +1,37 @@ +[alembic] +script_location = aodh.storage.sqlalchemy:alembic +sqlalchemy.url = + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = WARN +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/aodh/storage/sqlalchemy/alembic/env.py b/aodh/storage/sqlalchemy/alembic/env.py new file mode 100644 index 000000000..fd15eb5f9 --- /dev/null +++ b/aodh/storage/sqlalchemy/alembic/env.py @@ -0,0 +1,92 @@ +# +# Copyright 2015 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import with_statement +from alembic import context +from logging.config import fileConfig + +from aodh.storage import impl_sqlalchemy +from aodh.storage.sqlalchemy import models + + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = models.Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + conf = config.conf + context.configure(url=conf.database.connection, + target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + conf = config.conf + conn = impl_sqlalchemy.Connection(conf, conf.database.connection) + connectable = conn._engine_facade.get_engine() + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + conn.disconnect() + +if not hasattr(config, "conf"): + from aodh import service + config.conf = service.prepare_service([]) + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/aodh/storage/sqlalchemy/alembic/script.py.mako b/aodh/storage/sqlalchemy/alembic/script.py.mako new file mode 100644 index 000000000..c413ee393 --- /dev/null +++ b/aodh/storage/sqlalchemy/alembic/script.py.mako @@ -0,0 +1,39 @@ +# Copyright ${create_date.year} OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/aodh/storage/sqlalchemy/alembic/versions/12fe8fac9fe4_initial_base.py b/aodh/storage/sqlalchemy/alembic/versions/12fe8fac9fe4_initial_base.py new file mode 100644 index 000000000..dbff9e0b7 --- /dev/null +++ b/aodh/storage/sqlalchemy/alembic/versions/12fe8fac9fe4_initial_base.py @@ -0,0 +1,92 @@ +# Copyright 2015 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +"""initial base + +Revision ID: 12fe8fac9fe4 +Revises: +Create Date: 2015-07-28 17:38:37.022899 + +""" + +# revision identifiers, used by Alembic. +revision = '12fe8fac9fe4' +down_revision = None +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa + +import aodh.storage.sqlalchemy.models + + +def upgrade(): + op.create_table( + 'alarm_history', + sa.Column('event_id', sa.String(length=128), nullable=False), + sa.Column('alarm_id', sa.String(length=128), nullable=True), + sa.Column('on_behalf_of', sa.String(length=128), nullable=True), + sa.Column('project_id', sa.String(length=128), nullable=True), + sa.Column('user_id', sa.String(length=128), nullable=True), + sa.Column('type', sa.String(length=20), nullable=True), + sa.Column('detail', sa.Text(), nullable=True), + sa.Column('timestamp', + aodh.storage.sqlalchemy.models.PreciseTimestamp(), + nullable=True), + sa.PrimaryKeyConstraint('event_id') + ) + op.create_index( + 'ix_alarm_history_alarm_id', 'alarm_history', ['alarm_id'], + unique=False) + op.create_table( + 'alarm', + sa.Column('alarm_id', sa.String(length=128), nullable=False), + sa.Column('enabled', sa.Boolean(), nullable=True), + sa.Column('name', sa.Text(), nullable=True), + sa.Column('type', sa.String(length=50), nullable=True), + sa.Column('severity', sa.String(length=50), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('timestamp', + aodh.storage.sqlalchemy.models.PreciseTimestamp(), + nullable=True), + sa.Column('user_id', sa.String(length=128), nullable=True), + sa.Column('project_id', sa.String(length=128), nullable=True), + sa.Column('state', sa.String(length=255), nullable=True), + sa.Column('state_timestamp', + aodh.storage.sqlalchemy.models.PreciseTimestamp(), + nullable=True), + sa.Column('ok_actions', + aodh.storage.sqlalchemy.models.JSONEncodedDict(), + nullable=True), + sa.Column('alarm_actions', + aodh.storage.sqlalchemy.models.JSONEncodedDict(), + nullable=True), + sa.Column('insufficient_data_actions', + aodh.storage.sqlalchemy.models.JSONEncodedDict(), + nullable=True), + sa.Column('repeat_actions', sa.Boolean(), nullable=True), + sa.Column('rule', + aodh.storage.sqlalchemy.models.JSONEncodedDict(), + nullable=True), + sa.Column('time_constraints', + aodh.storage.sqlalchemy.models.JSONEncodedDict(), + nullable=True), + sa.PrimaryKeyConstraint('alarm_id') + ) + op.create_index( + 'ix_alarm_project_id', 'alarm', ['project_id'], unique=False) + op.create_index( + 'ix_alarm_user_id', 'alarm', ['user_id'], unique=False) diff --git a/aodh/storage/sqlalchemy/models.py b/aodh/storage/sqlalchemy/models.py index 74a69a7cd..688177b7d 100644 --- a/aodh/storage/sqlalchemy/models.py +++ b/aodh/storage/sqlalchemy/models.py @@ -53,7 +53,7 @@ class PreciseTimestamp(TypeDecorator): return dialect.type_descriptor(DECIMAL(precision=20, scale=6, asdecimal=True)) - return self.impl + return dialect.type_descriptor(self.impl) @staticmethod def process_bind_param(value, dialect): @@ -63,6 +63,11 @@ class PreciseTimestamp(TypeDecorator): return utils.dt_to_decimal(value) return value + def compare_against_backend(self, dialect, conn_type): + if dialect.name == 'mysql': + return issubclass(type(conn_type), DECIMAL) + return issubclass(type(conn_type), type(self.impl)) + @staticmethod def process_result_value(value, dialect): if value is None: diff --git a/aodh/tests/storage/sqlalchemy/test_migrations.py b/aodh/tests/storage/sqlalchemy/test_migrations.py new file mode 100644 index 000000000..f2aa54535 --- /dev/null +++ b/aodh/tests/storage/sqlalchemy/test_migrations.py @@ -0,0 +1,62 @@ +# +# Copyright 2015 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import abc + +import mock +from oslo_config import fixture as fixture_config +from oslo_db.sqlalchemy import test_migrations +import six +import six.moves.urllib.parse as urlparse + +from aodh import service +from aodh.storage import impl_sqlalchemy +from aodh.storage.sqlalchemy import models +from aodh.tests import base + + +class ABCSkip(base.SkipNotImplementedMeta, abc.ABCMeta): + pass + + +class ModelsMigrationsSync( + six.with_metaclass(ABCSkip, + base.BaseTestCase, + test_migrations.ModelsMigrationsSync)): + + def setUp(self): + super(ModelsMigrationsSync, self).setUp() + self.db = mock.Mock() + conf = service.prepare_service([]) + self.conf = self.useFixture(fixture_config.Config(conf)).conf + db_url = self.conf.database.connection + if not db_url: + self.skipTest("The db connection option should be specified.") + connection_scheme = urlparse.urlparse(db_url).scheme + engine_name = connection_scheme.split('+')[0] + if engine_name not in ('postgresql', 'mysql', 'sqlite'): + self.skipTest("This test only works with PostgreSQL or MySQL or" + " SQLite") + self.conn = impl_sqlalchemy.Connection(self.conf, + self.conf.database.connection) + + @staticmethod + def get_metadata(): + return models.Base.metadata + + def get_engine(self): + return self.conn._engine_facade.get_engine() + + def db_sync(self, engine): + self.conn.upgrade(nocreate=True) diff --git a/aodh/tests/storage/sqlalchemy/test_models.py b/aodh/tests/storage/sqlalchemy/test_models.py index 2a60750be..e9730f719 100644 --- a/aodh/tests/storage/sqlalchemy/test_models.py +++ b/aodh/tests/storage/sqlalchemy/test_models.py @@ -32,6 +32,8 @@ class PreciseTimestampTest(base.BaseTestCase): def _type_descriptor_mock(desc): if type(desc) == DECIMAL: return NUMERIC(precision=desc.precision, scale=desc.scale) + else: + return desc dialect = mock.MagicMock() dialect.name = name dialect.type_descriptor = _type_descriptor_mock diff --git a/doc/source/conf.py b/doc/source/conf.py index 501519e52..5f272fc16 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -85,7 +85,7 @@ copyright = u'2012-2015, OpenStack Foundation' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['**/#*', '**~', '**/#*#'] +exclude_patterns = ['**/#*', '**~', '**/#*#', '**/*alembic*'] # The reST default role (used for this markup: `text`) # to use for all documents. diff --git a/requirements.txt b/requirements.txt index b5a26a3bc..7518b1f50 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. +alembic>=0.7.2 retrying!=1.3.0,>=1.2.3 # Apache-2.0 croniter>=0.3.4 # MIT License eventlet>=0.17.4 diff --git a/setup.cfg b/setup.cfg index 558edec89..345916143 100644 --- a/setup.cfg +++ b/setup.cfg @@ -75,6 +75,7 @@ source-dir = doc/source [pbr] warnerrors = true autodoc_index_modules = true +autodoc_exclude_modules = aodh.storage.sqlalchemy.alembic.* [extract_messages] keywords = _ gettext ngettext l_ lazy_gettext