update migrate to alembic

Change-Id: I4c3b3b6925efa86d8d425dae5074cd12c531d394

requirements.txt

@@ -20,7 +20,7 @@ oslo.upgradecheck>=0.1.0 # Apache-2.0
 oslo.utils>=4.5.0 # Apache-2.0
 oslo.versionedobjects>=1.31.2 # Apache-2.0
 osprofiler>=3.4.0
-SQLAlchemy>=0.9.0,!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8 # MIT
+SQLAlchemy>=1.4.0 # MIT
 alembic>=0.8.10 # MIT
 stevedore>=1.5.0 # Apache-2.0
 keystonemiddleware>=4.17.0 # Apache-2.0

venus/cmd/manage.py

@@ -22,21 +22,19 @@ import os
 import sys
 
 from oslo_config import cfg
-from oslo_db.sqlalchemy import migration
 from oslo_log import log as logging
 
 from venus.conf import CONF
-from venus import i18n
-from venus.i18n import _
-from venus import objects
-from venus.task import timer
-from venus import version
 
 from venus import context
 from venus import db
-from venus.db import migration as db_migration
+from venus import i18n
-from venus.db.sqlalchemy import api as db_api
+from venus import objects
+from venus import version
 
+from venus.db import migration as db_migration
+from venus.i18n import _
+from venus.task import timer
 
 i18n.enable_lazy()
 
@@ -241,9 +239,7 @@ class DbCommands(object):
 
     def version(self):
         """Print the current database version."""
-        print(migration.db_version(db_api.get_engine(),
+        print(db_migration.db_version())
-                                   db_migration.MIGRATE_REPO_PATH,
-                                   db_migration.INIT_VERSION))
 
     @args('age_in_days', type=int,
           help='Purge deleted rows older than age in days')

venus/db/alembic.ini

@@ -0,0 +1,110 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location =  %(here)s/alembic
+
+# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
+# Uncomment the line below if you want the files to be prepended with date and time
+# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
+# for all available tokens
+# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = .
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the python-dateutil library that can be
+# installed by adding `alembic[tz]` to the pip requirements
+# string value is passed to dateutil.tz.gettz()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "version_path_separator" below.
+# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions
+
+# version path separator; As mentioned above, this is the character used to split
+# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
+# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
+# Valid values for version_path_separator are:
+#
+# version_path_separator = :
+# version_path_separator = ;
+# version_path_separator = space
+version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
+
+# set to 'true' to search source files recursively
+# in each "version_locations" directory
+# new in Alembic version 1.10
+# recursive_version_locations = false
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+sqlalchemy.url =  sqlite:///venus.db
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

venus/db/alembic/README

@@ -0,0 +1 @@
+Generic single-database configuration.

venus/db/alembic/env.py

@@ -0,0 +1,97 @@
+# Copyright 2020 Inspur
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+  CLI interface for venus management.
+"""
+
+
+from logging.config import fileConfig
+
+from sqlalchemy import engine_from_config
+from sqlalchemy import pool
+
+from alembic import context
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = None
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection, target_metadata=target_metadata
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

venus/db/alembic/script.py.mako

@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

venus/db/alembic/versions/a6cf98f55b4d_.py

@@ -0,0 +1,61 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Initial revision
+
+Revision ID: c6214ca60943
+Revises:
+Create Date: 2023-03-22 18:04:02.387269
+"""
+
+from alembic import op
+from oslo_utils import timeutils
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'a6cf98f55b4d'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    t_mo_regitster_task = op.create_table(
+        't_mo_regitster_task1',
+        sa.Column('Id', sa.Integer, nullable=False, primary_key=True),
+        sa.Column('task_name', sa.String(255), nullable=True,
+                  primary_key=False),
+        sa.Column('host_name', sa.String(255), nullable=True,
+                  primary_key=False),
+        sa.Column('created_at', sa.DateTime, nullable=True, primary_key=False),
+        sa.Column('updated_at', sa.DateTime, nullable=True, primary_key=False),
+        sa.Column('deleted', sa.String(1), nullable=True, primary_key=False),
+        sa.Column('deleted_at', sa.DateTime, nullable=True, primary_key=False),
+        mysql_engine='InnoDB',
+        mysql_charset='utf8'
+    )
+
+    now = timeutils.utcnow()
+    new_data = {
+        'Id': '1',
+        'task_name': 'delete_es_index',
+        'host_name': '',
+        'update_time': now
+    }
+
+    op.bulk_insert(
+        t_mo_regitster_task,
+        [
+            new_data,
+        ],
+    )

venus/db/api.py

@@ -0,0 +1,48 @@
+# Copyright 2020 Inspur
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Implementation of SQLAlchemy backend."""
+
+
+from oslo_db.sqlalchemy import enginefacade
+from oslo_log import log as logging
+from venus.conf import CONF
+
+
+_facade = None
+db_context = enginefacade.transaction_context()
+LOG = logging.getLogger(__name__)
+
+
+def get_facade():
+    global _facade
+    if _facade is None:
+
+        # FIXME: get_facade() is called by the test suite startup,
+        # but will not be called normally for API calls.
+        # osprofiler / oslo_db / enginefacade currently don't have hooks
+        # to talk to each other, however one needs to be added to oslo.db
+        # to allow access to the Engine once constructed.
+        db_context.configure(**CONF.database)
+        _facade = db_context.get_legacy_facade()
+
+    return _facade
+
+
+def get_engine():
+    return get_facade().get_engine()
+
+
+def get_session():
+    return get_facade().get_session()

venus/db/base.py

@@ -14,21 +14,12 @@
 
 """Base class for classes that need modular database access."""
 
-from oslo_config import cfg
 from oslo_utils import importutils
-
+from venus.conf import CONF
-
-db_driver_opt = cfg.StrOpt('db_driver',
-                           default='venus.db',
-                           help='Driver to use for database access')
-
-CONF = cfg.CONF
-CONF.register_opt(db_driver_opt)
 
 
 class Base(object):
     """DB driver is injected in the init method."""
-
     def __init__(self, db_driver=None):
         # NOTE(mriedem): Without this call, multiple inheritance involving
         # the db Base class does not work correctly.

venus/db/migration.py

@@ -1,4 +1,6 @@
-# Copyright 2020 Inspur
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
 #    not use this file except in compliance with the License. You may obtain
@@ -15,44 +17,88 @@
 """Database setup and migration commands."""
 
 import os
-import threading
 
-from oslo_config import cfg
+from alembic import command as alembic_api
+from alembic import config as alembic_config
+from alembic import migration as alembic_migration
 from oslo_db import options
-from stevedore import driver
+from oslo_log import log as logging
 
-from venus.db.sqlalchemy import api as db_api
+from venus.conf import CONF
+from venus.db import api as db_api
 
-INIT_VERSION = 000
+options.set_defaults(CONF)
 
-_IMPL = None
+LOG = logging.getLogger(__name__)
-_LOCK = threading.Lock()
-
-options.set_defaults(cfg.CONF)
-
-MIGRATE_REPO_PATH = os.path.join(
-    os.path.abspath(os.path.dirname(__file__)),
-    'sqlalchemy',
-    'migrate_repo',
-)
 
 
-def get_backend():
+def _find_alembic_conf():
-    global _IMPL
+    """Get the project's alembic configuration
-    if _IMPL is None:
+
-        with _LOCK:
+    :returns: An instance of ``alembic.config.Config``
-            if _IMPL is None:
+    """
-                _IMPL = driver.DriverManager(
+    path = os.path.join(
-                    "venus.database.migration_backend",
+        os.path.abspath(os.path.dirname(__file__)), 'alembic.ini')
-                    cfg.CONF.database.backend).driver
+
-    return _IMPL
+    config = alembic_config.Config(os.path.abspath(path))
+    # we don't want to use the logger configuration from the file, which is
+    # only really intended for the CLI
+    # https://stackoverflow.com/a/42691781/613428
+    config.attributes['configure_logger'] = True
+
+    return config
 
 
-def db_sync(version=None, init_version=INIT_VERSION, engine=None):
+def _upgrade_alembic(engine, config, version):
-    """Migrate the database to `version` or the most recent version."""
+    # re-use the connection rather than creating a new one
+    with engine.connect() as connection:
+        config.attributes['connection'] = connection
+        alembic_api.upgrade(config, version or 'head')
+
+
+def db_version():
+    """Get database version."""
+    engine = db_api.get_engine()
+
+    with engine.connect() as conn:
+        m_context = alembic_migration.MigrationContext.configure(conn)
+        return m_context.get_current_revision()
+
+
+def db_sync(version=None, engine=None):
+    """Migrate the database to `version` or the most recent version.
+
+    We're currently straddling two migration systems, sqlalchemy-migrate and
+    alembic. This handles both by ensuring we switch from one to the other at
+    the appropriate moment.
+    """
+
+    # if the user requested a specific version, check if it's an integer: if
+    # so, we're almost certainly in sqlalchemy-migrate land and won't support
+    # that
+    if version is not None and version.isdigit():
+        raise ValueError(
+            'You requested an sqlalchemy-migrate database version; this is '
+            'no longer supported'
+        )
+
     if engine is None:
         engine = db_api.get_engine()
-    return get_backend().db_sync(engine=engine,
+
-                                 abs_path=MIGRATE_REPO_PATH,
+    config = _find_alembic_conf()
-                                 version=version,
+
-                                 init_version=init_version)
+    # discard the URL encoded in alembic.ini in favour of the URL configured
+    # for the engine by the database fixtures, casting from
+    # 'sqlalchemy.engine.url.URL' to str in the process. This returns a
+    # RFC-1738 quoted URL, which means that a password like "foo@" will be
+    # turned into "foo%40". This in turns causes a problem for
+    # set_main_option() because that uses ConfigParser.set, which (by design)
+    # uses *python* interpolation to write the string out ... where "%" is the
+    # special python interpolation character! Avoid this mismatch by quoting
+    # all %'s for the set below.
+    engine_url = str(engine.url).replace('%', '%%')
+    config.set_main_option('sqlalchemy.url', str(engine_url))
+
+    LOG.info('Applying migration(s)')
+    _upgrade_alembic(engine, config, version)
+    LOG.info('Migration(s) applied')

venus/db/sqlalchemy/api.py

@@ -1,161 +0,0 @@
-# Copyright 2020 Inspur
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-"""Implementation of SQLAlchemy backend."""
-
-import sqlalchemy
-import sys
-import threading
-import warnings
-
-from oslo_config import cfg
-from oslo_db import options
-from oslo_db.sqlalchemy import session as db_session
-from oslo_log import log as logging
-import osprofiler.sqlalchemy
-
-from venus import exception
-from venus.i18n import _
-
-CONF = cfg.CONF
-CONF.import_group("profiler", "venus.service")
-log = logging.getLogger(__name__)
-
-options.set_defaults(CONF, connection='sqlite:///$state_path/venus.sqlite')
-
-_LOCK = threading.Lock()
-_FACADE = None
-
-
-def _create_facade_lazily():
-    global _LOCK
-    with _LOCK:
-        global _FACADE
-        if _FACADE is None:
-            _FACADE = db_session.EngineFacade(
-                CONF.database.connection,
-                **dict(CONF.database)
-            )
-
-            if CONF.profiler.profiler_enabled:
-                if CONF.profiler.trace_sqlalchemy:
-                    osprofiler.sqlalchemy.add_tracing(sqlalchemy,
-                                                      _FACADE.get_engine(),
-                                                      "db")
-
-        return _FACADE
-
-
-def get_engine():
-    facade = _create_facade_lazily()
-    return facade.get_engine()
-
-
-def get_session(**kwargs):
-    facade = _create_facade_lazily()
-    return facade.get_session(**kwargs)
-
-
-def dispose_engine():
-    get_engine().dispose()
-
-
-_DEFAULT_QUOTA_NAME = 'default'
-
-
-def get_backend():
-    """The backend is this module itself."""
-
-    return sys.modules[__name__]
-
-
-def is_admin_context(context):
-    """Indicates if the request context is an administrator."""
-    if not context:
-        warnings.warn(_('Use of empty request context is deprecated'),
-                      DeprecationWarning)
-        raise Exception('die')
-    return context.is_admin
-
-
-def is_user_context(context):
-    """Indicates if the request context is a normal user."""
-    if not context:
-        return False
-    if context.is_admin:
-        return False
-    if not context.user_id or not context.project_id:
-        return False
-    return True
-
-
-def authorize_project_context(context, project_id):
-    """Ensures a request has permission to access the given project."""
-    if is_user_context(context):
-        if not context.project_id:
-            raise exception.NotAuthorized()
-        elif context.project_id != project_id:
-            raise exception.NotAuthorized()
-
-
-def authorize_user_context(context, user_id):
-    """Ensures a request has permission to access the given user."""
-    if is_user_context(context):
-        if not context.user_id:
-            raise exception.NotAuthorized()
-        elif context.user_id != user_id:
-            raise exception.NotAuthorized()
-
-
-def authorize_quota_class_context(context, class_name):
-    """Ensures a request has permission to access the given quota class."""
-    if is_user_context(context):
-        if not context.quota_class:
-            raise exception.NotAuthorized()
-        elif context.quota_class != class_name:
-            raise exception.NotAuthorized()
-
-
-def require_admin_context(f):
-    """Decorator to require admin request context.
-
-    The first argument to the wrapped function must be the context.
-
-    """
-
-    def wrapper(*args, **kwargs):
-        if not is_admin_context(args[0]):
-            raise exception.AdminRequired()
-        return f(*args, **kwargs)
-
-    return wrapper
-
-
-def require_context(f):
-    """Decorator to require *any* user or admin context.
-
-    This does no authorization for user or project access matching, see
-    :py:func:`authorize_project_context` and
-    :py:func:`authorize_user_context`.
-
-    The first argument to the wrapped function must be the context.
-
-    """
-
-    def wrapper(*args, **kwargs):
-        if not is_admin_context(args[0]) and not is_user_context(args[0]):
-            raise exception.NotAuthorized()
-        return f(*args, **kwargs)
-
-    return wrapper

venus/modules/custom_config/backends/sql.py

@@ -15,7 +15,8 @@
 """Implementation of SQLAlchemy backend."""
 
 import time
-from venus.db.sqlalchemy.api import get_session
+
+from venus.db.api import get_session
 from venus.modules.custom_config.backends import models