openstack/vitrage
Code Issues Proposed changes

Merge "Add ability to upgrade db"

Browse Source
This commit is contained in:
Zuul 2019-09-11 07:40:21 +00:00 committed by Gerrit Code Review
commit c7f1ef0a93
16 changed files with 697 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
requirements.txt
View File

@ -3,6 +3,7 @@
# process, which may cause wedges in the gate later.
pbr>=3.1.1 # Apache-2.0
alembic>=0.9.8 # MIT
Babel>=2.5.3 # BSD
lxml>=4.1.1 # BSD
PyMySQL>=0.8.0 # MIT License

2
setup.cfg
View File

@ -32,6 +32,8 @@ console_scripts =
vitrage-persistor = vitrage.cli.persistor:main
vitrage-ml = vitrage.cli.machine_learning:main
vitrage-dbsync = vitrage.cli.storage:dbsync
vitrage-dbsync-revision = vitrage.cli.storage:revision
vitrage-dbsync-stamp = vitrage.cli.storage:stamp
vitrage-purge-data = vitrage.cli.storage:purge_data
vitrage-snmp-parsing = vitrage.cli.snmp_parsing:main
vitrage-status = vitrage.cli.status:main

35
vitrage/cli/storage.py
View File

@ -14,15 +14,48 @@
import sys
from oslo_config import cfg
from vitrage.cli import VITRAGE_TITLE
from vitrage.common import config
from vitrage import storage
from vitrage.storage.sqlalchemy import migration
CONF = cfg.CONF
CLI_OPTS = [
cfg.StrOpt('revision',
default='head',
help='Migration version')
]
REVISION_OPTS = [
cfg.StrOpt('message',
help='Text that will be used for migration title'),
cfg.BoolOpt('autogenerate',
default=False,
help='Generates diff based on current database state')
]
def stamp():
print(VITRAGE_TITLE)
CONF.register_cli_opts(CLI_OPTS)
config.parse_config(sys.argv)
migration.stamp(CONF.revision)
def revision():
print(VITRAGE_TITLE)
CONF.register_cli_opts(REVISION_OPTS)
config.parse_config(sys.argv)
migration.revision(CONF.message, CONF.autogenerate)
def dbsync():
print(VITRAGE_TITLE)
CONF.register_cli_opts(CLI_OPTS)
config.parse_config(sys.argv)
storage.get_connection_from_config().upgrade()
migration.upgrade(CONF.revision)
def purge_data():

4
vitrage/storage/base.py
View File

@ -55,10 +55,6 @@ class Connection(object):
def changes(self):
return None
@abc.abstractmethod
def upgrade(self, nocreate=False):
raise NotImplementedError('upgrade is not implemented')
@abc.abstractmethod
def disconnect(self):
raise NotImplementedError('disconnect is not implemented')

26
vitrage/storage/impl_sqlalchemy.py
View File

@ -106,32 +106,6 @@ class Connection(base.Connection):
return str(url)
return url
def upgrade(self, nocreate=False):
engine = self._engine_facade.get_engine()
engine.connect()
# As the following tables were changed in Rocky, they are removed and
# created. This is fine for an upgrade from Queens, since data in these
# was anyway deleted in each restart.
# starting From Rocky, data in these tables should not be removed.
models.Base.metadata.drop_all(
engine, tables=[
models.ActiveAction.__table__,
models.Event.__table__,
models.GraphSnapshot.__table__])
models.Base.metadata.create_all(
engine, tables=[models.ActiveAction.__table__,
models.Template.__table__,
models.Webhooks.__table__,
models.Event.__table__,
models.GraphSnapshot.__table__,
models.Alarm.__table__,
models.Edge.__table__,
models.Change.__table__])
# TODO(idan_hefetz) upgrade logic is missing
def disconnect(self):
self._engine_facade.get_engine().dispose()

91
vitrage/storage/sqlalchemy/migration/__init__.py Normal file
View File

@ -0,0 +1,91 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from alembic import command as alembic_command
from alembic import config as alembic_config
from alembic import migration as alembic_migrations
from oslo_config import cfg
from oslo_db import exception as db_exc
from oslo_db.sqlalchemy import enginefacade
from vitrage.storage.sqlalchemy import models
CONF = cfg.CONF
def _alembic_config():
path = os.path.join(os.path.dirname(__file__), 'alembic.ini')
config = alembic_config.Config(path)
return config
def create_schema(config=None, engine=None):
"""Create database schema from models description.
Can be used for initial installation instead of upgrade('head').
"""
if engine is None:
engine = enginefacade.writer.get_engine()
# NOTE(viktors): If we will use metadata.create_all() for non empty db
# schema, it will only add the new tables, but leave
# existing as is. So we should avoid of this situation.
if version(engine=engine) is not None:
raise db_exc.DBMigrationError("DB schema is already under version"
" control. Use upgrade() instead")
models.Base.metadata.create_all(engine)
stamp('head', config=config)
def revision(message=None, autogenerate=False, config=None):
"""Creates template for migration.
:param message: Text that will be used for migration title
:type message: string
:param autogenerate: If True - generates diff based on current database
state
:type autogenerate: bool
"""
config = config or _alembic_config()
return alembic_command.revision(config, message=message,
autogenerate=autogenerate)
def stamp(revision, config=None):
"""Stamps database with provided revision.
Don't run any migrations.
:param revision: Should match one from repository or head - to stamp
database with most recent revision
:type revision: string
"""
config = config or _alembic_config()
return alembic_command.stamp(config, revision=revision)
def upgrade(revision):
config = _alembic_config()
alembic_command.upgrade(config, revision)
def version(config=None, engine=None):
if engine is None:
engine = enginefacade.writer.get_engine()
with engine.connect() as conn:
context = alembic_migrations.MigrationContext.configure(conn)
return context.get_current_revision()

58
vitrage/storage/sqlalchemy/migration/alembic.ini Normal file
View File

@ -0,0 +1,58 @@
# A generic, single database configuration.
[alembic]
# path to migration scripts
script_location = %(here)s/alembic_migrations
# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s
# max length of characters to apply to the
# "slug" field
#truncate_slug_length = 40
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
# set to 'true' to allow .pyc and .pyo files without
# a source .py file to be detected as revisions in the
# versions/ directory
# sourceless = false
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S

18
vitrage/storage/sqlalchemy/migration/alembic_migrations/README Normal file
View File

@ -0,0 +1,18 @@
The migrations in `alembic_migrations/versions` contain the changes needed to
migrate between Vitrage database revisions. A migration occurs by executing a
script that details the changes needed to upgrade the database. The migration
scripts are ordered so that multiple scripts can run sequentially. The scripts
are executed by Vitrage's migration wrapper which uses the Alembic library to
manage the migration. Vitrage supports migration from Train release or later.
Please see https://alembic.readthedocs.org/en/latest/index.html for general documentation
To create alembic migrations use:
$ vitrage-dbsync-revision --message --autogenerate
Stamp db with most recent migration version, without actually running migrations
$ vitrage-dbsync-stamp --revision head
Upgrade can be performed by:
$ vitrage-dbsync - for backward compatibility
$ vitrage-dbsync --revision head

0
vitrage/storage/sqlalchemy/migration/alembic_migrations/__init__.py Normal file
View File

76
vitrage/storage/sqlalchemy/migration/alembic_migrations/env.py Normal file
View File

@ -0,0 +1,76 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import with_statement
from alembic import context
from logging.config import fileConfig
from oslo_db.sqlalchemy import enginefacade
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
target_metadata = None
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
url = config.get_main_option("sqlalchemy.url")
context.configure(
url=url, target_metadata=target_metadata, literal_binds=True)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
engine = enginefacade.writer.get_engine()
with engine.connect() as connection:
context.configure(connection=connection,
target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()

32
vitrage/storage/sqlalchemy/migration/alembic_migrations/script.py.mako Normal file
View File

@ -0,0 +1,32 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision}
Create Date: ${create_date}
"""
# revision identifiers, used by Alembic.
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
def upgrade():
${upgrades if upgrades else "pass"}

260
vitrage/storage/sqlalchemy/migration/alembic_migrations/versions/4e44c9414dff_initial_migration.py Normal file
View File

@ -0,0 +1,260 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from alembic import op
from oslo_utils import timeutils
import sqlalchemy as sa
from vitrage.storage.sqlalchemy import models
"""Initial migration"
Revision ID: 4e44c9414dff
Revises: None
Create Date: 2019-09-04 15:35:01.086784
"""
# revision identifiers, used by Alembic.
revision = '4e44c9414dff'
down_revision = None
def upgrade():
try:
op.create_table(
'alarms',
sa.Column('vitrage_id', sa.String(128), primary_key=True),
sa.Column('start_timestamp', sa.DateTime, nullable=False),
sa.Column('end_timestamp', sa.DateTime, nullable=False,
default=models.DEFAULT_END_TIME),
sa.Column('name', sa.String(256), nullable=False),
sa.Column('vitrage_type', sa.String(64), nullable=False),
sa.Column('vitrage_aggregated_severity', sa.String(64),
nullable=False),
sa.Column('vitrage_operational_severity', sa.String(64),
nullable=False),
sa.Column('project_id', sa.String(64)),
sa.Column('vitrage_resource_type', sa.String(64)),
sa.Column('vitrage_resource_id', sa.String(64)),
sa.Column('vitrage_resource_project_id', sa.String(64)),
sa.Column('payload', models.JSONEncodedDict),
sa.Column('created_at', sa.DateTime,
default=lambda: timeutils.utcnow()),
sa.Column('updated_at', sa.DateTime,
onupdate=lambda: timeutils.utcnow()),
mysql_charset='utf8',
mysql_engine='InnoDB'
)
op.create_table(
'edges',
sa.Column('source_id', sa.String(128), primary_key=True),
sa.Column('target_id', sa.String(128), primary_key=True),
sa.Column('label', sa.String(64), nullable=False),
sa.Column('start_timestamp', sa.DateTime, nullable=False),
sa.Column('end_timestamp', sa.DateTime, nullable=False,
default=models.DEFAULT_END_TIME),
sa.Column('payload', models.JSONEncodedDict),
sa.Column('created_at', sa.DateTime,
default=lambda: timeutils.utcnow()),
sa.Column('updated_at', sa.DateTime,
onupdate=lambda: timeutils.utcnow()),
sa.ForeignKeyConstraint(['source_id'], ['alarms.vitrage_id'],
ondelete='CASCADE'),
sa.ForeignKeyConstraint(['target_id'], ['alarms.vitrage_id'],
ondelete='CASCADE'),
mysql_charset='utf8',
mysql_engine='InnoDB'
)
op.create_table(
'changes',
sa.Column('id', models.MagicBigInt, primary_key=True,
autoincrement=True),
sa.Column('vitrage_id', sa.String(128), nullable=False),
sa.Column('timestamp', sa.DateTime, nullable=False),
sa.Column('severity', sa.String(64), nullable=False),
sa.Column('payload', models.JSONEncodedDict),
sa.Column('created_at', sa.DateTime,
default=lambda: timeutils.utcnow()),
sa.Column('updated_at', sa.DateTime,
onupdate=lambda: timeutils.utcnow()),
sa.ForeignKeyConstraint(['vitrage_id'], ['alarms.vitrage_id'],
ondelete='CASCADE'),
mysql_charset='utf8',
mysql_engine='InnoDB'
)
op.create_table(
'active_actions',
sa.Column('action_type', sa.String(128)),
sa.Column('extra_info', sa.String(128)),
sa.Column('source_vertex_id', sa.String(128)),
sa.Column('target_vertex_id', sa.String(128)),
sa.Column('action_id', sa.String(128), primary_key=True),
sa.Column('score', sa.SmallInteger()),
sa.Column('trigger', sa.String(128), primary_key=True),
sa.Column('created_at', sa.DateTime,
default=lambda: timeutils.utcnow()),
sa.Column('updated_at', sa.DateTime,
onupdate=lambda: timeutils.utcnow()),
mysql_charset='utf8',
mysql_engine='InnoDB'
)
op.create_table(
'templates',
sa.Column('id', sa.String(64), primary_key=True, nullable=False),
sa.Column('status', sa.String(16)),
sa.Column('status_details', sa.String(128)),
sa.Column('name', sa.String(128), nullable=False),
sa.Column('file_content', models.JSONEncodedDict, nullable=False),
sa.Column("type", sa.String(64), default='standard'),
sa.Column('created_at', sa.DateTime,
default=lambda: timeutils.utcnow()),
sa.Column('updated_at', sa.DateTime,
onupdate=lambda: timeutils.utcnow()),
mysql_charset='utf8',
mysql_engine='InnoDB'
)
op.create_table(
'webhooks',
sa.Column('id', sa.String(128), primary_key=True),
sa.Column('project_id', sa.String(128), nullable=False),
sa.Column('is_admin_webhook', sa.Boolean, nullable=False),
sa.Column('url', sa.String(256), nullable=False),
sa.Column('headers', sa.String(1024)),
sa.Column('regex_filter', sa.String(512)),
sa.Column('created_at', sa.DateTime,
default=lambda: timeutils.utcnow()),
sa.Column('updated_at', sa.DateTime,
onupdate=lambda: timeutils.utcnow()),
mysql_charset='utf8',
mysql_engine='InnoDB'
)
op.create_index(
'ix_active_action',
'active_actions',
[
'action_type', 'extra_info', 'source_vertex_id',
'target_vertex_id'
]
)
op.create_table(
'events',
sa.Column("id", sa.BigInteger, primary_key=True, nullable=False,
autoincrement=True),
sa.Column('payload', models.JSONEncodedDict(), nullable=False),
sa.Column('is_vertex', sa.Boolean, nullable=False),
sa.Column('created_at', sa.DateTime,
default=lambda: timeutils.utcnow()),
sa.Column('updated_at', sa.DateTime,
onupdate=lambda: timeutils.utcnow()),
mysql_charset='utf8',
mysql_engine='InnoDB'
)
op.create_table(
'graph_snapshots',
sa.Column('id', sa.Integer, primary_key=True,
nullable=False),
sa.Column('event_id', sa.BigInteger, nullable=False),
sa.Column('graph_snapshot', models.CompressedBinary((2 ** 32) - 1),
nullable=False),
sa.Column('created_at', sa.DateTime,
default=lambda: timeutils.utcnow()),
sa.Column('updated_at', sa.DateTime,
onupdate=lambda: timeutils.utcnow()),
mysql_charset='utf8',
mysql_engine='InnoDB'
)
op.create_index(
'ix_alarms_end_timestamp',
'alarms',
[
'end_timestamp'
]
)
op.create_index(
'ix_alarms_project_id',
'alarms',
[
'project_id'
]
)
op.create_index(
'ix_alarms_start_timestamp',
'alarms',
[
'start_timestamp'
]
)
op.create_index(
'ix_alarms_vitrage_aggregated_severity',
'alarms',
[
'vitrage_aggregated_severity'
]
)
op.create_index(
'ix_alarms_vitrage_operational_severity',
'alarms',
[
'vitrage_operational_severity'
]
)
op.create_index(
'ix_alarms_vitrage_resource_project_id',
'alarms',
[
'vitrage_resource_project_id'
]
)
op.create_index(
'ix_changes_severity',
'changes',
[
'severity'
]
)
op.create_index(
'ix_changes_timestamp',
'changes',
[
'timestamp'
]
)
op.create_index(
'ix_changes_vitrage_id',
'changes',
[
'vitrage_id'
]
)
except Exception:
# TODO(e0ne): figure out more specific exception here to handle a case
# when migration is applied over Queens release and tables are already
# exists
pass

0
vitrage/storage/sqlalchemy/migration/alembic_migrations/versions/__init__.py Normal file
View File

6
vitrage/storage/sqlalchemy/models.py
View File

@ -110,7 +110,7 @@ class Event(Base):
)
class ActiveAction(Base, models.TimestampMixin):
class ActiveAction(Base):
__tablename__ = 'active_actions'
__table_args__ = (
# Index 'ix_active_action' on fields:
@ -170,7 +170,7 @@ class GraphSnapshot(Base):
)
class Template(Base, models.TimestampMixin):
class Template(Base):
__tablename__ = 'templates'
uuid = Column("id", String(64), primary_key=True, nullable=False)
@ -210,6 +210,7 @@ class Webhooks(Base):
"<Webhook(" \
"id='%s', " \
"created_at='%s', " \
"updated_at='%s', " \
"project_id='%s', " \
"is_admin_webhook='%s', " \
"url='%s', " \
@ -218,6 +219,7 @@ class Webhooks(Base):
(
self.id,
self.created_at,
self.updated_at,
self.project_id,
self.is_admin_webhook,
self.url,

0
vitrage/tests/unit/storage/__init__.py Normal file
View File

121
vitrage/tests/unit/storage/test_migrations.py Normal file
View File

@ -0,0 +1,121 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from alembic import script
import contextlib
import mock
from oslo_db.sqlalchemy import enginefacade
from oslo_db.sqlalchemy import test_fixtures
from oslo_db.sqlalchemy import test_migrations
from oslo_log import log as logging
from oslo_utils import excutils
from oslotest import base as test_base
from vitrage.storage.sqlalchemy import migration
from vitrage.storage.sqlalchemy import models
LOG = logging.getLogger(__name__)
@contextlib.contextmanager
def patch_with_engine(engine):
with mock.patch.object(enginefacade.writer, 'get_engine') as patch_engine:
patch_engine.return_value = engine
yield
class WalkWersionsMixin(object):
def _walk_versions(self, engine=None, alembic_cfg=None):
with patch_with_engine(engine):
script_directory = script.ScriptDirectory.from_config(alembic_cfg)
self.assertIsNone(self.migration_api.version(alembic_cfg))
versions = [ver for ver in script_directory.walk_revisions()]
for version in reversed(versions):
self._migrate_up(engine, alembic_cfg,
version.revision, with_data=True)
def _migrate_up(self, engine, config, version, with_data=False):
"""migrate up to a new version of the db.
We allow for data insertion and post checks at every
migration version with special _pre_upgrade_### and
_check_### functions in the main test.
"""
try:
if with_data:
data = None
pre_upgrade = getattr(
self, "_pre_upgrade_%s" % version, None)
if pre_upgrade:
data = pre_upgrade(engine)
self.migration_api.upgrade(version, config=config)
self.assertEqual(version, self.migration_api.version(config))
if with_data:
check = getattr(self, '_check_%s' % version, None)
if check:
check(engine, data)
except Exception:
excutils.save_and_reraise_exception(logger=LOG)
class MigrationCheckersMixin(object):
def setUp(self):
super(MigrationCheckersMixin, self).setUp()
self.engine = enginefacade.writer.get_engine()
self.config = migration._alembic_config()
self.migration_api = migration
def test_walk_versions(self):
self._walk_versions(self.engine, self.config)
def test_upgrade_and_version(self):
with patch_with_engine(self.engine):
self.migration_api.upgrade('head')
self.assertIsNotNone(self.migration_api.version())
class TestMigrationsMySQL(MigrationCheckersMixin,
WalkWersionsMixin,
test_fixtures.OpportunisticDBTestMixin):
FIXTURE = test_fixtures.MySQLOpportunisticFixture
class ModelsMigrationSyncMixin(object):
def setUp(self):
super(ModelsMigrationSyncMixin, self).setUp()
self.engine = enginefacade.writer.get_engine()
def get_metadata(self):
return models.Base.metadata
def get_engine(self):
return self.engine
def db_sync(self, engine):
with patch_with_engine(engine):
migration.upgrade('head')
class ModelsMigrationsMySQL(ModelsMigrationSyncMixin,
test_migrations.ModelsMigrationsSync,
test_fixtures.OpportunisticDBTestMixin,
test_base.BaseTestCase):
FIXTURE = test_fixtures.MySQLOpportunisticFixture