openstack/venus
Code Issues Proposed changes

update migrate to alembic

Browse Source 
Change-Id: I4c3b3b6925efa86d8d425dae5074cd12c531d394
This commit is contained in:
Frank Jia (贾思瑞) 2023-06-08 16:40:19 +08:00
parent 5e94c56872
commit 10721f164a
12 changed files with 438 additions and 224 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
requirements.txt
View File

@ -20,7 +20,7 @@ oslo.upgradecheck>=0.1.0 # Apache-2.0
oslo.utils>=4.5.0 # Apache-2.0 oslo.utils>=4.5.0 # Apache-2.0
oslo.versionedobjects>=1.31.2 # Apache-2.0 oslo.versionedobjects>=1.31.2 # Apache-2.0
osprofiler>=3.4.0 osprofiler>=3.4.0
SQLAlchemy>=0.9.0,!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8 # MIT SQLAlchemy>=1.4.0 # MIT
alembic>=0.8.10 # MIT alembic>=0.8.10 # MIT
stevedore>=1.5.0 # Apache-2.0 stevedore>=1.5.0 # Apache-2.0
keystonemiddleware>=4.17.0 # Apache-2.0 keystonemiddleware>=4.17.0 # Apache-2.0

18
venus/cmd/manage.py
View File

@ -22,21 +22,19 @@ import os
import sys import sys
from oslo_config import cfg from oslo_config import cfg
from oslo_db.sqlalchemy import migration
from oslo_log import log as logging from oslo_log import log as logging
from venus.conf import CONF from venus.conf import CONF
from venus import i18n
from venus.i18n import _
from venus import objects
from venus.task import timer
from venus import version
from venus import context from venus import context
from venus import db from venus import db
from venus.db import migration as db_migration from venus import i18n
from venus.db.sqlalchemy import api as db_api from venus import objects
from venus import version
from venus.db import migration as db_migration
from venus.i18n import _
from venus.task import timer
i18n.enable_lazy() i18n.enable_lazy()
@ -241,9 +239,7 @@ class DbCommands(object):
def version(self): def version(self):
"""Print the current database version.""" """Print the current database version."""
print(migration.db_version(db_api.get_engine(), print(db_migration.db_version())
db_migration.MIGRATE_REPO_PATH,
db_migration.INIT_VERSION))
@args('age_in_days', type=int, @args('age_in_days', type=int,
help='Purge deleted rows older than age in days') help='Purge deleted rows older than age in days')

110
venus/db/alembic.ini Normal file
View File

@ -0,0 +1,110 @@
# A generic, single database configuration.
[alembic]
# path to migration scripts
script_location = %(here)s/alembic
# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
# Uncomment the line below if you want the files to be prepended with date and time
# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
# for all available tokens
# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
# sys.path path, will be prepended to sys.path if present.
# defaults to the current working directory.
prepend_sys_path = .
# timezone to use when rendering the date within the migration file
# as well as the filename.
# If specified, requires the python-dateutil library that can be
# installed by adding `alembic[tz]` to the pip requirements
# string value is passed to dateutil.tz.gettz()
# leave blank for localtime
# timezone =
# max length of characters to apply to the
# "slug" field
# truncate_slug_length = 40
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
# set to 'true' to allow .pyc and .pyo files without
# a source .py file to be detected as revisions in the
# versions/ directory
# sourceless = false
# version location specification; This defaults
# to alembic/versions. When using multiple version
# directories, initial revisions must be specified with --version-path.
# The path separator used here should be the separator specified by "version_path_separator" below.
# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions
# version path separator; As mentioned above, this is the character used to split
# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
# Valid values for version_path_separator are:
#
# version_path_separator = :
# version_path_separator = ;
# version_path_separator = space
version_path_separator = os # Use os.pathsep. Default configuration used for new projects.
# set to 'true' to search source files recursively
# in each "version_locations" directory
# new in Alembic version 1.10
# recursive_version_locations = false
# the output encoding used when revision files
# are written from script.py.mako
# output_encoding = utf-8
sqlalchemy.url = sqlite:///venus.db
[post_write_hooks]
# post_write_hooks defines scripts or Python functions that are run
# on newly generated revision scripts. See the documentation for further
# detail and examples
# format using "black" - use the console_scripts runner, against the "black" entrypoint
# hooks = black
# black.type = console_scripts
# black.entrypoint = black
# black.options = -l 79 REVISION_SCRIPT_FILENAME
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S

1
venus/db/alembic/README Normal file
View File

@ -0,0 +1 @@
Generic single-database configuration.

97
venus/db/alembic/env.py Normal file
View File

@ -0,0 +1,97 @@
# Copyright 2020 Inspur
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
CLI interface for venus management.
"""
from logging.config import fileConfig
from sqlalchemy import engine_from_config
from sqlalchemy import pool
from alembic import context
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
if config.config_file_name is not None:
fileConfig(config.config_file_name)
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
target_metadata = None
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
def run_migrations_offline() -> None:
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
url = config.get_main_option("sqlalchemy.url")
context.configure(
url=url,
target_metadata=target_metadata,
literal_binds=True,
dialect_opts={"paramstyle": "named"},
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online() -> None:
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
connectable = engine_from_config(
config.get_section(config.config_ini_section, {}),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
with connectable.connect() as connection:
context.configure(
connection=connection, target_metadata=target_metadata
)
with context.begin_transaction():
context.run_migrations()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()

24
venus/db/alembic/script.py.mako Normal file
View File

@ -0,0 +1,24 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
# revision identifiers, used by Alembic.
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
branch_labels = ${repr(branch_labels)}
depends_on = ${repr(depends_on)}
def upgrade() -> None:
${upgrades if upgrades else "pass"}
def downgrade() -> None:
${downgrades if downgrades else "pass"}

61
venus/db/alembic/versions/a6cf98f55b4d_.py Normal file
View File

@ -0,0 +1,61 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Initial revision
Revision ID: c6214ca60943
Revises:
Create Date: 2023-03-22 18:04:02.387269
"""
from alembic import op
from oslo_utils import timeutils
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'a6cf98f55b4d'
down_revision = None
branch_labels = None
depends_on = None
def upgrade() -> None:
t_mo_regitster_task = op.create_table(
't_mo_regitster_task1',
sa.Column('Id', sa.Integer, nullable=False, primary_key=True),
sa.Column('task_name', sa.String(255), nullable=True,
primary_key=False),
sa.Column('host_name', sa.String(255), nullable=True,
primary_key=False),
sa.Column('created_at', sa.DateTime, nullable=True, primary_key=False),
sa.Column('updated_at', sa.DateTime, nullable=True, primary_key=False),
sa.Column('deleted', sa.String(1), nullable=True, primary_key=False),
sa.Column('deleted_at', sa.DateTime, nullable=True, primary_key=False),
mysql_engine='InnoDB',
mysql_charset='utf8'
)
now = timeutils.utcnow()
new_data = {
'Id': '1',
'task_name': 'delete_es_index',
'host_name': '',
'update_time': now
}
op.bulk_insert(
t_mo_regitster_task,
[
new_data,
],
)

48
venus/db/api.py Normal file
View File

@ -0,0 +1,48 @@
# Copyright 2020 Inspur
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Implementation of SQLAlchemy backend."""
from oslo_db.sqlalchemy import enginefacade
from oslo_log import log as logging
from venus.conf import CONF
_facade = None
db_context = enginefacade.transaction_context()
LOG = logging.getLogger(__name__)
def get_facade():
global _facade
if _facade is None:
# FIXME: get_facade() is called by the test suite startup,
# but will not be called normally for API calls.
# osprofiler / oslo_db / enginefacade currently don't have hooks
# to talk to each other, however one needs to be added to oslo.db
# to allow access to the Engine once constructed.
db_context.configure(**CONF.database)
_facade = db_context.get_legacy_facade()
return _facade
def get_engine():
return get_facade().get_engine()
def get_session():
return get_facade().get_session()

11
venus/db/base.py
View File

@ -14,21 +14,12 @@
"""Base class for classes that need modular database access.""" """Base class for classes that need modular database access."""
from oslo_config import cfg
from oslo_utils import importutils from oslo_utils import importutils
from venus.conf import CONF
db_driver_opt = cfg.StrOpt('db_driver',
default='venus.db',
help='Driver to use for database access')
CONF = cfg.CONF
CONF.register_opt(db_driver_opt)
class Base(object): class Base(object):
"""DB driver is injected in the init method.""" """DB driver is injected in the init method."""
def __init__(self, db_driver=None): def __init__(self, db_driver=None):
# NOTE(mriedem): Without this call, multiple inheritance involving # NOTE(mriedem): Without this call, multiple inheritance involving
# the db Base class does not work correctly. # the db Base class does not work correctly.

126
venus/db/migration.py
View File

@ -1,58 +1,104 @@
# Copyright 2020 Inspur # Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may # Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain # not use this file except in compliance with the License. You may obtain
# a copy of the License at # a copy of the License at
# #
# http://www.apache.org/licenses/LICENSE-2.0 # http://www.apache.org/licenses/LICENSE-2.0
# #
# Unless required by applicable law or agreed to in writing, software # Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
"""Database setup and migration commands.""" """Database setup and migration commands."""
import os import os
import threading
from oslo_config import cfg from alembic import command as alembic_api
from alembic import config as alembic_config
from alembic import migration as alembic_migration
from oslo_db import options from oslo_db import options
from stevedore import driver from oslo_log import log as logging
from venus.db.sqlalchemy import api as db_api from venus.conf import CONF
from venus.db import api as db_api
INIT_VERSION = 000 options.set_defaults(CONF)
_IMPL = None LOG = logging.getLogger(__name__)
_LOCK = threading.Lock()
options.set_defaults(cfg.CONF)
MIGRATE_REPO_PATH = os.path.join(
os.path.abspath(os.path.dirname(__file__)),
'sqlalchemy',
'migrate_repo',
)
def get_backend(): def _find_alembic_conf():
global _IMPL """Get the project's alembic configuration
if _IMPL is None:
with _LOCK: :returns: An instance of ``alembic.config.Config``
if _IMPL is None: """
_IMPL = driver.DriverManager( path = os.path.join(
"venus.database.migration_backend", os.path.abspath(os.path.dirname(__file__)), 'alembic.ini')
cfg.CONF.database.backend).driver
return _IMPL config = alembic_config.Config(os.path.abspath(path))
# we don't want to use the logger configuration from the file, which is
# only really intended for the CLI
# https://stackoverflow.com/a/42691781/613428
config.attributes['configure_logger'] = True
return config
def db_sync(version=None, init_version=INIT_VERSION, engine=None): def _upgrade_alembic(engine, config, version):
"""Migrate the database to `version` or the most recent version.""" # re-use the connection rather than creating a new one
with engine.connect() as connection:
config.attributes['connection'] = connection
alembic_api.upgrade(config, version or 'head')
def db_version():
"""Get database version."""
engine = db_api.get_engine()
with engine.connect() as conn:
m_context = alembic_migration.MigrationContext.configure(conn)
return m_context.get_current_revision()
def db_sync(version=None, engine=None):
"""Migrate the database to `version` or the most recent version.
We're currently straddling two migration systems, sqlalchemy-migrate and
alembic. This handles both by ensuring we switch from one to the other at
the appropriate moment.
"""
# if the user requested a specific version, check if it's an integer: if
# so, we're almost certainly in sqlalchemy-migrate land and won't support
# that
if version is not None and version.isdigit():
raise ValueError(
'You requested an sqlalchemy-migrate database version; this is '
'no longer supported'
)
if engine is None: if engine is None:
engine = db_api.get_engine() engine = db_api.get_engine()
return get_backend().db_sync(engine=engine,
abs_path=MIGRATE_REPO_PATH, config = _find_alembic_conf()
version=version,
init_version=init_version) # discard the URL encoded in alembic.ini in favour of the URL configured
# for the engine by the database fixtures, casting from
# 'sqlalchemy.engine.url.URL' to str in the process. This returns a
# RFC-1738 quoted URL, which means that a password like "foo@" will be
# turned into "foo%40". This in turns causes a problem for
# set_main_option() because that uses ConfigParser.set, which (by design)
# uses *python* interpolation to write the string out ... where "%" is the
# special python interpolation character! Avoid this mismatch by quoting
# all %'s for the set below.
engine_url = str(engine.url).replace('%', '%%')
config.set_main_option('sqlalchemy.url', str(engine_url))
LOG.info('Applying migration(s)')
_upgrade_alembic(engine, config, version)
LOG.info('Migration(s) applied')

161
venus/db/sqlalchemy/api.py
View File

@ -1,161 +0,0 @@
# Copyright 2020 Inspur
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Implementation of SQLAlchemy backend."""
import sqlalchemy
import sys
import threading
import warnings
from oslo_config import cfg
from oslo_db import options
from oslo_db.sqlalchemy import session as db_session
from oslo_log import log as logging
import osprofiler.sqlalchemy
from venus import exception
from venus.i18n import _
CONF = cfg.CONF
CONF.import_group("profiler", "venus.service")
log = logging.getLogger(__name__)
options.set_defaults(CONF, connection='sqlite:///$state_path/venus.sqlite')
_LOCK = threading.Lock()
_FACADE = None
def _create_facade_lazily():
global _LOCK
with _LOCK:
global _FACADE
if _FACADE is None:
_FACADE = db_session.EngineFacade(
CONF.database.connection,
**dict(CONF.database)
)
if CONF.profiler.profiler_enabled:
if CONF.profiler.trace_sqlalchemy:
osprofiler.sqlalchemy.add_tracing(sqlalchemy,
_FACADE.get_engine(),
"db")
return _FACADE
def get_engine():
facade = _create_facade_lazily()
return facade.get_engine()
def get_session(**kwargs):
facade = _create_facade_lazily()
return facade.get_session(**kwargs)
def dispose_engine():
get_engine().dispose()
_DEFAULT_QUOTA_NAME = 'default'
def get_backend():
"""The backend is this module itself."""
return sys.modules[__name__]
def is_admin_context(context):
"""Indicates if the request context is an administrator."""
if not context:
warnings.warn(_('Use of empty request context is deprecated'),
DeprecationWarning)
raise Exception('die')
return context.is_admin
def is_user_context(context):
"""Indicates if the request context is a normal user."""
if not context:
return False
if context.is_admin:
return False
if not context.user_id or not context.project_id:
return False
return True
def authorize_project_context(context, project_id):
"""Ensures a request has permission to access the given project."""
if is_user_context(context):
if not context.project_id:
raise exception.NotAuthorized()
elif context.project_id != project_id:
raise exception.NotAuthorized()
def authorize_user_context(context, user_id):
"""Ensures a request has permission to access the given user."""
if is_user_context(context):
if not context.user_id:
raise exception.NotAuthorized()
elif context.user_id != user_id:
raise exception.NotAuthorized()
def authorize_quota_class_context(context, class_name):
"""Ensures a request has permission to access the given quota class."""
if is_user_context(context):
if not context.quota_class:
raise exception.NotAuthorized()
elif context.quota_class != class_name:
raise exception.NotAuthorized()
def require_admin_context(f):
"""Decorator to require admin request context.
The first argument to the wrapped function must be the context.
"""
def wrapper(*args, **kwargs):
if not is_admin_context(args[0]):
raise exception.AdminRequired()
return f(*args, **kwargs)
return wrapper
def require_context(f):
"""Decorator to require *any* user or admin context.
This does no authorization for user or project access matching, see
:py:func:`authorize_project_context` and
:py:func:`authorize_user_context`.
The first argument to the wrapped function must be the context.
"""
def wrapper(*args, **kwargs):
if not is_admin_context(args[0]) and not is_user_context(args[0]):
raise exception.NotAuthorized()
return f(*args, **kwargs)
return wrapper

3
venus/modules/custom_config/backends/sql.py
View File

@ -15,7 +15,8 @@
"""Implementation of SQLAlchemy backend.""" """Implementation of SQLAlchemy backend."""
import time import time
from venus.db.sqlalchemy.api import get_session
from venus.db.api import get_session
from venus.modules.custom_config.backends import models from venus.modules.custom_config.backends import models