From fb8da5d535546754904cd26f2664f455ba724acf Mon Sep 17 00:00:00 2001 From: Fei Long Wang Date: Wed, 9 Nov 2016 10:09:14 +1300 Subject: [PATCH] Add support for sqlalchemy migration based on alembic Implement blueprint: sqlalchemy-migration Change-Id: Ifd0d5186299611907a388f8ab8b27e312a3cb60d --- ...sqlalchemy-migration-6b4eaebb6e02a449.yaml | 3 + requirements.txt | 1 + setup.cfg | 1 + test-requirements.txt | 2 + .../storage/sqlalchemy/migration/__init__.py | 0 .../storage/sqlalchemy/migration/alembic.ini | 54 +++++ .../migration/alembic_migrations/README.md | 73 +++++++ .../migration/alembic_migrations/env.py | 96 +++++++++ .../alembic_migrations/script.py.mako | 34 ++++ .../versions/001_liberty.py | 72 +++++++ .../versions/002_placeholder.py | 30 +++ .../versions/003_placeholder.py | 30 +++ .../versions/004_placeholder.py | 30 +++ .../versions/005_placeholder.py | 30 +++ zaqar/storage/sqlalchemy/migration/cli.py | 118 +++++++++++ .../storage/sqlalchemy_migration/__init__.py | 0 .../test_db_manage_cli.py | 89 +++++++++ .../sqlalchemy_migration/test_migrations.py | 175 ++++++++++++++++ .../test_migrations_base.py | 188 ++++++++++++++++++ 19 files changed, 1026 insertions(+) create mode 100644 releasenotes/notes/sqlalchemy-migration-6b4eaebb6e02a449.yaml create mode 100644 zaqar/storage/sqlalchemy/migration/__init__.py create mode 100644 zaqar/storage/sqlalchemy/migration/alembic.ini create mode 100644 zaqar/storage/sqlalchemy/migration/alembic_migrations/README.md create mode 100644 zaqar/storage/sqlalchemy/migration/alembic_migrations/env.py create mode 100644 zaqar/storage/sqlalchemy/migration/alembic_migrations/script.py.mako create mode 100644 zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/001_liberty.py create mode 100644 zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/002_placeholder.py create mode 100644 zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/003_placeholder.py create mode 100644 zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/004_placeholder.py create mode 100644 zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/005_placeholder.py create mode 100644 zaqar/storage/sqlalchemy/migration/cli.py create mode 100644 zaqar/tests/unit/storage/sqlalchemy_migration/__init__.py create mode 100644 zaqar/tests/unit/storage/sqlalchemy_migration/test_db_manage_cli.py create mode 100644 zaqar/tests/unit/storage/sqlalchemy_migration/test_migrations.py create mode 100644 zaqar/tests/unit/storage/sqlalchemy_migration/test_migrations_base.py diff --git a/releasenotes/notes/sqlalchemy-migration-6b4eaebb6e02a449.yaml b/releasenotes/notes/sqlalchemy-migration-6b4eaebb6e02a449.yaml new file mode 100644 index 000000000..8df730ef9 --- /dev/null +++ b/releasenotes/notes/sqlalchemy-migration-6b4eaebb6e02a449.yaml @@ -0,0 +1,3 @@ +--- +features: + - Add migration support for Zaqar's sqlalchemy storage driver. diff --git a/requirements.txt b/requirements.txt index bf82ffa65..65706e4e1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ # process, which may cause wedges in the gate later. pbr>=1.6 # Apache-2.0 +alembic>=0.8.4 # MIT Babel>=2.3.4 # BSD falcon>=0.1.6 # Apache-2.0 jsonschema!=2.5.0,<3.0.0,>=2.0.0 # MIT diff --git a/setup.cfg b/setup.cfg index fa401838a..24acec1b6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,6 +32,7 @@ console_scripts = zaqar-bench = zaqar.bench.conductor:main zaqar-server = zaqar.cmd.server:run zaqar-gc = zaqar.cmd.gc:run + zaqar-sql-db-manage = zaqar.storage.sqlalchemy.migration.cli:main zaqar.data.storage = mongodb = zaqar.storage.mongodb.driver:DataDriver diff --git a/test-requirements.txt b/test-requirements.txt index cd3792ab6..81f4a37fb 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -19,6 +19,8 @@ fixtures>=3.0.0 # Apache-2.0/BSD python-subunit>=0.0.18 # Apache-2.0/BSD testrepository>=0.0.18 # Apache-2.0/BSD testtools>=1.4.0 # MIT +oslo.db>=4.11.0,!=4.13.1,!=4.13.2 # Apache-2.0 +testresources>=0.2.4 # Apache-2.0/BSD # Documentation sphinx!=1.3b1,<1.4,>=1.2.1 # BSD diff --git a/zaqar/storage/sqlalchemy/migration/__init__.py b/zaqar/storage/sqlalchemy/migration/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/zaqar/storage/sqlalchemy/migration/alembic.ini b/zaqar/storage/sqlalchemy/migration/alembic.ini new file mode 100644 index 000000000..2e8b2f41f --- /dev/null +++ b/zaqar/storage/sqlalchemy/migration/alembic.ini @@ -0,0 +1,54 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = zaqar/storage/sqlalchemy/migration/alembic_migrations + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +sqlalchemy.url = + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/zaqar/storage/sqlalchemy/migration/alembic_migrations/README.md b/zaqar/storage/sqlalchemy/migration/alembic_migrations/README.md new file mode 100644 index 000000000..b2c77a15d --- /dev/null +++ b/zaqar/storage/sqlalchemy/migration/alembic_migrations/README.md @@ -0,0 +1,73 @@ + + +The migrations in `alembic_migrations/versions` contain the changes needed to migrate +between Zaqar database revisions. A migration occurs by executing a script that +details the changes needed to upgrade the database. The migration scripts +are ordered so that multiple scripts can run sequentially. The scripts are executed by +Zaqar's migration wrapper which uses the Alembic library to manage the migration. Zaqar +supports migration from Liberty or later. + +You can upgrade to the latest database version via: +``` +$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf upgrade head +``` + +To check the current database version: +``` +$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf current +``` + +To create a script to run the migration offline: +``` +$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf upgrade head --sql +``` + +To run the offline migration between specific migration versions: +``` +$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf upgrade : --sql +``` + +Upgrade the database incrementally: +``` +$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf upgrade --delta <# of revs> +``` + +Create new revision: +``` +$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf revision -m "description of revision" --autogenerate +``` + +Create a blank file: +``` +$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf revision -m "description of revision" +``` + +This command does not perform any migrations, it only sets the revision. +Revision may be any existing revision. Use this command carefully. +``` +$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf stamp +``` + +To verify that the timeline does branch, you can run this command: +``` +$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf check_migration +``` + +If the migration path does branch, you can find the branch point via: +``` +$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf history +``` diff --git a/zaqar/storage/sqlalchemy/migration/alembic_migrations/env.py b/zaqar/storage/sqlalchemy/migration/alembic_migrations/env.py new file mode 100644 index 000000000..7a667c26e --- /dev/null +++ b/zaqar/storage/sqlalchemy/migration/alembic_migrations/env.py @@ -0,0 +1,96 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Based on Neutron's migration/cli.py + +from __future__ import with_statement +from logging import config as c + +from alembic import context +from oslo_utils import importutils +from sqlalchemy import create_engine +from sqlalchemy import pool + +from zaqar.storage.sqlalchemy import tables + + +importutils.try_import('zaqar.storage.sqlalchemy.tables') + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config +zaqar_config = config.zaqar_config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +c.fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = tables.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + context.configure( + url=zaqar_config['drivers:management_store:sqlalchemy'].uri) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + engine = create_engine( + zaqar_config['drivers:management_store:sqlalchemy'].uri, + poolclass=pool.NullPool) + + connection = engine.connect() + context.configure( + connection=connection, + target_metadata=target_metadata) + + try: + with context.begin_transaction(): + context.run_migrations() + finally: + connection.close() + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/zaqar/storage/sqlalchemy/migration/alembic_migrations/script.py.mako b/zaqar/storage/sqlalchemy/migration/alembic_migrations/script.py.mako new file mode 100644 index 000000000..f70210ddd --- /dev/null +++ b/zaqar/storage/sqlalchemy/migration/alembic_migrations/script.py.mako @@ -0,0 +1,34 @@ +# Copyright ${create_date.year} OpenStack Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + + +def upgrade(): + ${upgrades if upgrades else "pass"} diff --git a/zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/001_liberty.py b/zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/001_liberty.py new file mode 100644 index 000000000..4f0d9e2f9 --- /dev/null +++ b/zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/001_liberty.py @@ -0,0 +1,72 @@ +# Copyright 2016 OpenStack Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Liberty release + +Revision ID: 001 +Revises: None +Create Date: 2015-09-13 20:46:25.783444 + +""" + +# revision identifiers, used by Alembic. +revision = '001' +down_revision = None + +from alembic import op +import sqlalchemy as sa + +MYSQL_ENGINE = 'InnoDB' +MYSQL_CHARSET = 'utf8' + + +def upgrade(): + op.create_table('queues', + sa.Column('id', sa.INTEGER, primary_key=True), + sa.Column('project', sa.String(64)), + sa.Column('name', sa.String(64)), + sa.Column('metadata', sa.LargeBinary), + sa.UniqueConstraint('project', 'name')) + + op.create_table('poolgroup', + sa.Column('name', sa.String(64), primary_key=True)) + + op.create_table('pools', + sa.Column('name', sa.String(64), primary_key=True), + sa.Column('group', sa.String(64), + sa.ForeignKey('poolgroup.name', + ondelete='CASCADE'), + nullable=True), + sa.Column('uri', sa.String(255), + unique=True, nullable=False), + sa.Column('weight', sa.INTEGER, nullable=False), + sa.Column('options', sa.Text())) + + op.create_table('flavors', + sa.Column('name', sa.String(64), primary_key=True), + sa.Column('project', sa.String(64)), + sa.Column('pool_group', sa.String(64), + sa.ForeignKey('poolgroup.name', + ondelete='CASCADE'), + nullable=False), + sa.Column('capabilities', sa.Text())) + + op.create_table('catalogue', + sa.Column('pool', sa.String(64), + sa.ForeignKey('pools.name', + ondelete='CASCADE')), + sa.Column('project', sa.String(64)), + sa.Column('queue', sa.String(64), nullable=False), + sa.UniqueConstraint('project', 'queue')) diff --git a/zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/002_placeholder.py b/zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/002_placeholder.py new file mode 100644 index 000000000..12590ce4f --- /dev/null +++ b/zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/002_placeholder.py @@ -0,0 +1,30 @@ +# Copyright 2016 OpenStack Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""placeholder + +Revision ID: 002 +Revises: 001 +Create Date: 2014-04-01 21:04:47.941098 + +""" + +# revision identifiers, used by Alembic. +revision = '002' +down_revision = '001' + + +def upgrade(): + pass diff --git a/zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/003_placeholder.py b/zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/003_placeholder.py new file mode 100644 index 000000000..5bd06bd50 --- /dev/null +++ b/zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/003_placeholder.py @@ -0,0 +1,30 @@ +# Copyright 2014 OpenStack Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""placeholder + +Revision ID: 003 +Revises: 002 +Create Date: 2014-04-01 21:05:00.270366 + +""" + +# revision identifiers, used by Alembic. +revision = '003' +down_revision = '002' + + +def upgrade(): + pass diff --git a/zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/004_placeholder.py b/zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/004_placeholder.py new file mode 100644 index 000000000..434dac75e --- /dev/null +++ b/zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/004_placeholder.py @@ -0,0 +1,30 @@ +# Copyright 2014 OpenStack Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""placeholder + +Revision ID: 004 +Revises: 003 +Create Date: 2014-04-01 21:04:57.627883 + +""" + +# revision identifiers, used by Alembic. +revision = '004' +down_revision = '003' + + +def upgrade(): + pass diff --git a/zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/005_placeholder.py b/zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/005_placeholder.py new file mode 100644 index 000000000..c8f4f876e --- /dev/null +++ b/zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/005_placeholder.py @@ -0,0 +1,30 @@ +# Copyright 2014 OpenStack Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""placeholder + +Revision ID: 005 +Revises: 004 +Create Date: 2014-04-01 21:04:54.928605 + +""" + +# revision identifiers, used by Alembic. +revision = '005' +down_revision = '004' + + +def upgrade(): + pass diff --git a/zaqar/storage/sqlalchemy/migration/cli.py b/zaqar/storage/sqlalchemy/migration/cli.py new file mode 100644 index 000000000..061303ce6 --- /dev/null +++ b/zaqar/storage/sqlalchemy/migration/cli.py @@ -0,0 +1,118 @@ +# Copyright (c) 2016 Catalyst IT Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from alembic import command as alembic_cmd +from alembic import config as alembic_cfg +from alembic import util as alembic_u +from oslo_config import cfg + +CONF = cfg.CONF + + +def do_alembic_command(config, cmd, *args, **kwargs): + try: + getattr(alembic_cmd, cmd)(config, *args, **kwargs) + except alembic_u.CommandError as e: + alembic_u.err(str(e)) + + +def do_check_migration(config, _cmd): + do_alembic_command(config, 'branches') + + +def do_upgrade_downgrade(config, cmd): + if not CONF.command.revision and not CONF.command.delta: + raise SystemExit('You must provide a revision or relative delta') + + revision = CONF.command.revision + + if CONF.command.delta: + sign = '+' if CONF.command.name == 'upgrade' else '-' + revision = sign + str(CONF.command.delta) + + do_alembic_command(config, cmd, revision, sql=CONF.command.sql) + + +def do_stamp(config, cmd): + do_alembic_command(config, cmd, + CONF.command.revision, + sql=CONF.command.sql) + + +def do_revision(config, cmd): + do_alembic_command(config, cmd, + message=CONF.command.message, + autogenerate=CONF.command.autogenerate, + sql=CONF.command.sql) + + +def add_command_parsers(subparsers): + for name in ['current', 'history', 'branches']: + parser = subparsers.add_parser(name) + parser.set_defaults(func=do_alembic_command) + + parser = subparsers.add_parser('check_migration') + parser.set_defaults(func=do_check_migration) + + for name in ['upgrade', 'downgrade']: + parser = subparsers.add_parser(name) + parser.add_argument('--delta', type=int) + parser.add_argument('--sql', action='store_true') + parser.add_argument('revision', nargs='?') + parser.set_defaults(func=do_upgrade_downgrade) + + parser = subparsers.add_parser('stamp') + parser.add_argument('--sql', action='store_true') + parser.add_argument('revision') + parser.set_defaults(func=do_stamp) + + parser = subparsers.add_parser('revision') + parser.add_argument('-m', '--message') + parser.add_argument('--autogenerate', action='store_true') + parser.add_argument('--sql', action='store_true') + parser.set_defaults(func=do_revision) + + +command_opt = cfg.SubCommandOpt('command', + title='Command', + help='Available commands', + handler=add_command_parsers) + +CONF.register_cli_opt(command_opt) + +sqlalchemy_opts = [cfg.StrOpt('uri', + help='The SQLAlchemy connection string to' + ' use to connect to the database.', + secret=True)] + +CONF.register_opts(sqlalchemy_opts, + group='drivers:management_store:sqlalchemy') + + +def main(): + config = alembic_cfg.Config( + os.path.join(os.path.dirname(__file__), 'alembic.ini') + ) + config.set_main_option('script_location', + 'zaqar.storage.sqlalchemy.' + 'migration:alembic_migrations') + + # attach the octavia conf to the Alembic conf + config.zaqar_config = CONF + + CONF(project='zaqar') + CONF.command.func(config, CONF.command.name) diff --git a/zaqar/tests/unit/storage/sqlalchemy_migration/__init__.py b/zaqar/tests/unit/storage/sqlalchemy_migration/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/zaqar/tests/unit/storage/sqlalchemy_migration/test_db_manage_cli.py b/zaqar/tests/unit/storage/sqlalchemy_migration/test_db_manage_cli.py new file mode 100644 index 000000000..e776f7ed9 --- /dev/null +++ b/zaqar/tests/unit/storage/sqlalchemy_migration/test_db_manage_cli.py @@ -0,0 +1,89 @@ +# Copyright 2012 New Dream Network, LLC (DreamHost) +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys + +import mock +import testscenarios +import testtools + +from zaqar.storage.sqlalchemy.migration import cli + + +class TestCli(testtools.TestCase): + func_name = '' + exp_args = () + exp_kwargs = {} + + scenarios = [ + ('stamp', + dict(argv=['prog', 'stamp', 'foo'], func_name='stamp', + exp_args=('foo',), exp_kwargs={'sql': False})), + ('stamp-sql', + dict(argv=['prog', 'stamp', 'foo', '--sql'], func_name='stamp', + exp_args=('foo',), exp_kwargs={'sql': True})), + ('current', + dict(argv=['prog', 'current'], func_name='current', + exp_args=[], exp_kwargs=dict())), + ('history', + dict(argv=['prog', 'history'], func_name='history', + exp_args=[], exp_kwargs=dict())), + ('check_migration', + dict(argv=['prog', 'check_migration'], func_name='branches', + exp_args=[], exp_kwargs=dict())), + ('sync_revision_autogenerate', + dict(argv=['prog', 'revision', '--autogenerate', '-m', 'message'], + func_name='revision', + exp_args=(), + exp_kwargs={ + 'message': 'message', 'sql': False, 'autogenerate': True})), + ('sync_revision_sql', + dict(argv=['prog', 'revision', '--sql', '-m', 'message'], + func_name='revision', + exp_args=(), + exp_kwargs={ + 'message': 'message', 'sql': True, 'autogenerate': False})), + ('upgrade-sql', + dict(argv=['prog', 'upgrade', '--sql', 'head'], + func_name='upgrade', + exp_args=('head',), + exp_kwargs={'sql': True})), + + ('upgrade-delta', + dict(argv=['prog', 'upgrade', '--delta', '3'], + func_name='upgrade', + exp_args=('+3',), + exp_kwargs={'sql': False})) + ] + + def setUp(self): + super(TestCli, self).setUp() + do_alembic_cmd_p = mock.patch.object(cli, 'do_alembic_command') + self.addCleanup(do_alembic_cmd_p.stop) + self.do_alembic_cmd = do_alembic_cmd_p.start() + self.addCleanup(cli.CONF.reset) + + def test_cli(self): + with mock.patch.object(sys, 'argv', self.argv): + cli.main() + self.do_alembic_cmd.assert_has_calls( + [mock.call( + mock.ANY, self.func_name, + *self.exp_args, **self.exp_kwargs)] + ) + + +def load_tests(loader, in_tests, pattern): + return testscenarios.load_tests_apply_scenarios(loader, in_tests, pattern) diff --git a/zaqar/tests/unit/storage/sqlalchemy_migration/test_migrations.py b/zaqar/tests/unit/storage/sqlalchemy_migration/test_migrations.py new file mode 100644 index 000000000..37a6916b3 --- /dev/null +++ b/zaqar/tests/unit/storage/sqlalchemy_migration/test_migrations.py @@ -0,0 +1,175 @@ +# Copyright 2014 OpenStack Foundation +# Copyright 2014 Mirantis Inc +# Copyright 2016 Catalyst IT Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Tests for database migrations. + +For the opportunistic testing you need to set up a db named 'openstack_citest' +with user 'openstack_citest' and password 'openstack_citest' on localhost. +The test will then use that db and u/p combo to run the tests. + +For postgres on Ubuntu this can be done with the following commands: + +sudo -u postgres psql +postgres=# create user openstack_citest with createdb login password + 'openstack_citest'; +postgres=# create database openstack_citest with owner openstack_citest; + +""" + +import os + +from oslo_db.sqlalchemy import test_base +from oslo_db.sqlalchemy import utils as db_utils + +from zaqar.tests.unit.storage.sqlalchemy_migration import \ + test_migrations_base as base + + +class ZaqarMigrationsCheckers(object): + + def assertColumnExists(self, engine, table, column): + t = db_utils.get_table(engine, table) + self.assertIn(column, t.c) + + def assertColumnsExist(self, engine, table, columns): + for column in columns: + self.assertColumnExists(engine, table, column) + + def assertColumnType(self, engine, table, column, column_type): + t = db_utils.get_table(engine, table) + column_ref_type = str(t.c[column].type) + self.assertEqual(column_ref_type, column_type) + + def assertColumnCount(self, engine, table, columns): + t = db_utils.get_table(engine, table) + self.assertEqual(len(columns), len(t.columns)) + + def assertColumnNotExists(self, engine, table, column): + t = db_utils.get_table(engine, table) + self.assertNotIn(column, t.c) + + def assertIndexExists(self, engine, table, index): + t = db_utils.get_table(engine, table) + index_names = [idx.name for idx in t.indexes] + self.assertIn(index, index_names) + + def assertIndexMembers(self, engine, table, index, members): + self.assertIndexExists(engine, table, index) + + t = db_utils.get_table(engine, table) + index_columns = None + for idx in t.indexes: + if idx.name == index: + index_columns = idx.columns.keys() + break + + self.assertEqual(sorted(members), sorted(index_columns)) + + def test_walk_versions(self): + self.walk_versions(self.engine) + + def _pre_upgrade_001(self, engine): + # Anything returned from this method will be + # passed to corresponding _check_xxx method as 'data'. + pass + + def _check_001(self, engine, data): + queues_columns = [ + 'id', + 'name', + 'project', + 'metadata' + ] + self.assertColumnsExist( + engine, 'queues', queues_columns) + self.assertColumnCount( + engine, 'queues', queues_columns) + + poolgroup_columns = [ + 'name', + ] + self.assertColumnsExist( + engine, 'poolgroup', poolgroup_columns) + self.assertColumnCount( + engine, 'poolgroup', poolgroup_columns) + + pools_columns = [ + 'name', + 'group', + 'uri', + 'weight', + 'options', + ] + self.assertColumnsExist( + engine, 'pools', pools_columns) + self.assertColumnCount( + engine, 'pools', pools_columns) + + flavors_columns = [ + 'name', + 'project', + 'pool_group', + 'capabilities', + ] + self.assertColumnsExist( + engine, 'flavors', flavors_columns) + self.assertColumnCount( + engine, 'flavors', flavors_columns) + + catalogue_columns = [ + 'pool', + 'project', + 'queue', + ] + self.assertColumnsExist( + engine, 'catalogue', catalogue_columns) + self.assertColumnCount( + engine, 'catalogue', catalogue_columns) + + self._data_001(engine, data) + + def _data_001(self, engine, data): + datasize = 512 * 1024 # 512kB + data = os.urandom(datasize) + t = db_utils.get_table(engine, 'job_binary_internal') + engine.execute(t.insert(), data=data, id='123', name='name') + new_data = engine.execute(t.select()).fetchone().data + self.assertEqual(data, new_data) + engine.execute(t.delete()) + + def _check_002(self, engine, data): + # currently, 002 is just a placeholder + pass + + def _check_003(self, engine, data): + # currently, 003 is just a placeholder + pass + + def _check_004(self, engine, data): + # currently, 004 is just a placeholder + pass + + def _check_005(self, engine, data): + # currently, 005 is just a placeholder + pass + + +class TestMigrationsMySQL(ZaqarMigrationsCheckers, + base.BaseWalkMigrationTestCase, + base.TestModelsMigrationsSync, + test_base.MySQLOpportunisticTestCase): + pass diff --git a/zaqar/tests/unit/storage/sqlalchemy_migration/test_migrations_base.py b/zaqar/tests/unit/storage/sqlalchemy_migration/test_migrations_base.py new file mode 100644 index 000000000..2bba5f6fb --- /dev/null +++ b/zaqar/tests/unit/storage/sqlalchemy_migration/test_migrations_base.py @@ -0,0 +1,188 @@ +# Copyright 2010-2011 OpenStack Foundation +# Copyright 2012-2013 IBM Corp. +# Copyright 2016 Catalyst IT Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# +# Ripped off from Nova's test_migrations.py +# The only difference between Nova and this code is usage of alembic instead +# of sqlalchemy migrations. +# +# There is an ongoing work to extact similar code to oslo incubator. Once it is +# extracted we'll be able to remove this file and use oslo. + +import io +import os +import sqlalchemy as sa + +import alembic +from alembic import command +from alembic import config as alembic_config +from alembic import migration +from alembic import script as alembic_script +from oslo_config import cfg +from oslo_db.sqlalchemy import test_migrations as t_m +from oslo_log import log as logging + +from zaqar.i18n import _LE +import zaqar.storage.sqlalchemy.migration +from zaqar.storage.sqlalchemy import tables + + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF + + +class BaseWalkMigrationTestCase(object): + + ALEMBIC_CONFIG = alembic_config.Config( + os.path.join( + os.path.dirname(zaqar.storage.sqlalchemy.migration.__file__), + 'alembic.ini') + ) + + ALEMBIC_CONFIG.zaqar_config = CONF + + def _configure(self, engine): + """For each type of repository we should do some of configure steps. + + For migrate_repo we should set under version control our database. + For alembic we should configure database settings. For this goal we + should use oslo_config and openstack.commom.db.sqlalchemy.session with + database functionality (reset default settings and session cleanup). + """ + CONF.set_override('uri', str(engine.url), + group='drivers:management_store:sqlalchemy', + enforce_type=True) + sa.cleanup() + + def _alembic_command(self, alembic_command, engine, *args, **kwargs): + """Most of alembic command return data into output. + + We should redefine this setting for getting info. + """ + self.ALEMBIC_CONFIG.stdout = buf = io.StringIO() + CONF.set_override('uri', str(engine.url), + group='drivers:management_store:sqlalchemy', + enforce_type=True) + sa.cleanup() + getattr(command, alembic_command)(*args, **kwargs) + res = buf.getvalue().strip() + LOG.debug('Alembic command {command} returns: {result}'.format( + command=alembic_command, result=res)) + sa.cleanup() + return res + + def _get_versions(self): + """Stores a list of versions. + + Since alembic version has a random algorithm of generation + (SA-migrate has an ordered autoincrement naming) we should store + a list of versions (version for upgrade) + for successful testing of migrations in up mode. + """ + + env = alembic_script.ScriptDirectory.from_config(self.ALEMBIC_CONFIG) + versions = [] + for rev in env.walk_revisions(): + versions.append(rev.revision) + + versions.reverse() + return versions + + def walk_versions(self, engine=None): + # Determine latest version script from the repo, then + # upgrade from 1 through to the latest, with no data + # in the databases. This just checks that the schema itself + # upgrades successfully. + + self._configure(engine) + versions = self._get_versions() + for ver in versions: + self._migrate_up(engine, ver, with_data=True) + + def _get_version_from_db(self, engine): + """Returns latest version from db for each type of migrate repo.""" + + conn = engine.connect() + try: + context = migration.MigrationContext.configure(conn) + version = context.get_current_revision() or '-1' + finally: + conn.close() + return version + + def _migrate(self, engine, version, cmd): + """Base method for manipulation with migrate repo. + + It will upgrade or downgrade the actual database. + """ + + self._alembic_command(cmd, engine, self.ALEMBIC_CONFIG, version) + + def _migrate_up(self, engine, version, with_data=False): + """migrate up to a new version of the db. + + We allow for data insertion and post checks at every + migration version with special _pre_upgrade_### and + _check_### functions in the main test. + """ + # NOTE(sdague): try block is here because it's impossible to debug + # where a failed data migration happens otherwise + check_version = version + try: + if with_data: + data = None + pre_upgrade = getattr( + self, "_pre_upgrade_%s" % check_version, None) + if pre_upgrade: + data = pre_upgrade(engine) + self._migrate(engine, version, 'upgrade') + self.assertEqual(version, self._get_version_from_db(engine)) + if with_data: + check = getattr(self, "_check_%s" % check_version, None) + if check: + check(engine, data) + except Exception: + LOG.error(_LE("Failed to migrate to version {version} on engine " + "{engine}").format(version=version, engine=engine)) + raise + + +class TestModelsMigrationsSync(t_m.ModelsMigrationsSync): + """Class for comparison of DB migration scripts and models. + + Allows to check if the DB schema obtained by applying of migration + scripts is equal to the one produced from models definitions. + """ + + ALEMBIC_CONFIG = alembic_config.Config( + os.path.join( + os.path.dirname(zaqar.storage.sqlalchemy.migration.__file__), + 'alembic.ini') + ) + ALEMBIC_CONFIG.zaqar_config = CONF + + def get_engine(self): + return self.engine + + def db_sync(self, engine): + CONF.set_override('uri', str(engine.url), + group='drivers:management_store:sqlalchemy', + enforce_type=True) + alembic.command.upgrade(self.ALEMBIC_CONFIG, 'head') + + def get_metadata(self): + return tables.metadata