diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 69e90bdbd..7183887b1 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -133,6 +133,15 @@ function disable_vitrage_datasource { } +# Set configuration for database backend. +function vitrage_configure_db_backend { + if [ "$VITRAGE_DATABASE" = 'mysql' ] || [ "$VITRAGE_DATABASE" = 'postgresql' ] ; then + iniset $VITRAGE_CONF database connection $(database_connection_url vitrage) + else + die $LINENO "Unable to configure unknown VITRAGE_DATABASE $VITRAGE_DATABASE" + fi +} + # Configure Vitrage function configure_vitrage { iniset_rpc_backend vitrage $VITRAGE_CONF @@ -165,6 +174,9 @@ function configure_vitrage { iniset $VITRAGE_CONF service_credentials region_name $REGION_NAME iniset $VITRAGE_CONF service_credentials auth_url $KEYSTONE_SERVICE_URI + # Configured db + vitrage_configure_db_backend + # remove neutron vitrage datasource if neutron datasource not installed if ! is_service_enabled neutron; then disable_vitrage_datasource neutron.network neutron.port @@ -239,6 +251,14 @@ function configure_vitrage { function init_vitrage { # Get vitrage keystone settings in place _vitrage_create_accounts + + # Create and upgrade database only when used + if is_service_enabled mysql postgresql; then + if [ "$VITRAGE_DATABASE" = 'mysql' ] || [ "$VITRAGE_DATABASE" = 'postgresql' ] ; then + recreate_database vitrage + $VITRAGE_BIN_DIR/vitrage-dbsync + fi + fi # Create cache dir sudo install -d -o $STACK_USER $VITRAGE_AUTH_CACHE_DIR rm -f $VITRAGE_AUTH_CACHE_DIR/* diff --git a/devstack/settings b/devstack/settings index e368511f9..0a6bddd10 100644 --- a/devstack/settings +++ b/devstack/settings @@ -18,6 +18,9 @@ VITRAGE_CONF=$VITRAGE_CONF_DIR/vitrage.conf VITRAGE_AUTH_CACHE_DIR=${VITRAGE_AUTH_CACHE_DIR:-/var/cache/vitrage} VITRAGE_WSGI_DIR=${VITRAGE_WSGI_DIR:-/var/www/vitrage} +# Set up database backend +VITRAGE_DATABASE=${VITRAGE_DATABASE:-mysql} + # Vitrage connection info. VITRAGE_SERVICE_PROTOCOL=http VITRAGE_SERVICE_HOST=$SERVICE_HOST diff --git a/etc/vitrage/vitrage-config-generator.conf b/etc/vitrage/vitrage-config-generator.conf index 9ebc83b99..91ea1691e 100644 --- a/etc/vitrage/vitrage-config-generator.conf +++ b/etc/vitrage/vitrage-config-generator.conf @@ -9,3 +9,4 @@ namespace = oslo.middleware namespace = oslo.policy namespace = keystonemiddleware.auth_token namespace = osprofiler +namespace = oslo.db diff --git a/requirements.txt b/requirements.txt index ed1e25cfb..89c40db77 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ pbr!=2.1.0,>=2.0.0 # Apache-2.0 Babel!=2.4.0,>=2.3.4 # BSD lxml!=3.7.0,>=3.4.1 # BSD +PyMySQL>=0.7.6 # MIT License python-ceilometerclient>=2.5.0 # Apache-2.0 python-cinderclient>=3.2.0 # Apache-2.0 python-dateutil>=2.4.2 # BSD @@ -16,6 +17,7 @@ pyzabbix>=0.7.4 # LGPL networkx>=1.10 # BSD oslo.config>=4.6.0 # Apache-2.0 oslo.context>=2.14.0 # Apache-2.0 +oslo.db>=4.24.0 # Apache-2.0 oslo.messaging>=5.29.0 # Apache-2.0 oslo.middleware>=3.31.0 # Apache-2.0 oslo.serialization!=2.19.1,>=2.18.0 # Apache-2.0 @@ -29,6 +31,7 @@ Werkzeug>=0.7 # BSD License keystonemiddleware>=4.17.0 # Apache-2.0 stevedore>=1.20.0 # Apache-2.0 voluptuous>=0.8.9 # BSD License +SQLAlchemy>=1.0.10,!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8 # MIT sympy>=0.7.6 # BSD pysnmp>=4.2.3 # BSD PyJWT>=1.0.1 # MIT diff --git a/setup.cfg b/setup.cfg index cf0dcc62d..f4a478be1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,6 +30,7 @@ console_scripts = vitrage-notifier = vitrage.cli.notifier:main vitrage-collector = vitrage.cli.collector:main vitrage-ml = vitrage.cli.machine_learning:main + vitrage-dbsync = vitrage.cli.storage:dbsync vitrage.entity_graph = networkx = vitrage.graph.driver.networkx_graph:NXGraph @@ -41,6 +42,12 @@ oslo.config.opts = tempest.test_plugins = vitrage_tests = vitrage_tempest_tests.plugin:VitrageTempestPlugin +vitrage.storage = + mysql = vitrage.storage.impl_sqlalchemy:Connection + mysql+pymysql = vitrage.storage.impl_sqlalchemy:Connection + postgresql = vitrage.storage.impl_sqlalchemy:Connection + sqlite = vitrage.storage.impl_sqlalchemy:Connection + [files] packages = vitrage diff --git a/test-requirements.txt b/test-requirements.txt index 4be7fb772..4a38a7fb7 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -7,6 +7,7 @@ python-dateutil>=2.4.2 # BSD coverage!=4.4,>=4.0 # Apache-2.0 lxml!=3.7.0,>=3.4.1 # BSD networkx>=1.10 # BSD +PyMySQL>=0.7.6 # MIT License python-ceilometerclient>=2.5.0 # Apache-2.0 python-cinderclient>=3.2.0 # Apache-2.0 python-neutronclient>=6.3.0 # Apache-2.0 @@ -14,6 +15,7 @@ python-novaclient>=9.1.0 # Apache-2.0 python-heatclient>=1.10.0 # Apache-2.0 python-subunit>=0.0.18 # Apache-2.0/BSD pyzabbix>=0.7.4 # LGPL +oslo.db>=4.24.0 # Apache-2.0 oslo.log>=3.30.0 # Apache-2.0 oslotest>=1.10.0 # Apache-2.0 oslo.service>=1.24.0 # Apache-2.0 @@ -33,6 +35,7 @@ sympy>=0.7.6 # BSD reno>=2.5.0 # Apache-2.0 pysnmp>=4.2.3 # BSD osprofiler>=1.4.0 # Apache-2.0 +SQLAlchemy>=1.0.10,!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8 # MIT # Doc requirements openstackdocstheme>=1.17.0 # Apache-2.0 diff --git a/vitrage/api/app.py b/vitrage/api/app.py index 70ecffb1d..6035f84cf 100644 --- a/vitrage/api/app.py +++ b/vitrage/api/app.py @@ -39,7 +39,8 @@ def setup_app(root, conf=None): app_hooks = [hooks.ConfigHook(conf), hooks.TranslationHook(), hooks.RPCHook(conf), - hooks.ContextHook()] + hooks.ContextHook(), + hooks.DBHook(conf)] app = pecan.make_app( root, diff --git a/vitrage/api/hooks.py b/vitrage/api/hooks.py index 6c965352e..224ebe7e2 100644 --- a/vitrage/api/hooks.py +++ b/vitrage/api/hooks.py @@ -18,6 +18,7 @@ from pecan import hooks from vitrage import messaging from vitrage import rpc as vitrage_rpc +from vitrage import storage class ConfigHook(hooks.PecanHook): @@ -74,3 +75,12 @@ class ContextHook(hooks.PecanHook): # Inject the context... state.request.context = ctx.to_dict() + + +class DBHook(hooks.PecanHook): + + def __init__(self, conf): + self.storage = storage.get_connection_from_config(conf) + + def before(self, state): + state.request.storage = self.storage diff --git a/vitrage/cli/graph.py b/vitrage/cli/graph.py index 34d31a00e..78882837f 100644 --- a/vitrage/cli/graph.py +++ b/vitrage/cli/graph.py @@ -61,6 +61,10 @@ def init(conf): 'Entity Graph', '%s:%s:%s' % (EntityCategory.RESOURCE, OPENSTACK_CLUSTER, CLUSTER_ID), uuid=True) + + # TODO(ihefetz) uncomment db connection creation + # db_connection = storage.get_connection_from_config(conf) + scenario_repo = ScenarioRepository(conf) evaluator = ScenarioEvaluator(conf, e_graph, scenario_repo, evaluator_q) diff --git a/vitrage/cli/storage.py b/vitrage/cli/storage.py new file mode 100644 index 000000000..693ad0093 --- /dev/null +++ b/vitrage/cli/storage.py @@ -0,0 +1,21 @@ +# Copyright 2017 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from vitrage import service +from vitrage import storage + + +def dbsync(): + conf = service.prepare_service() + storage.get_connection_from_config(conf).upgrade() diff --git a/vitrage/opts.py b/vitrage/opts.py index ea1462064..ee84f887a 100644 --- a/vitrage/opts.py +++ b/vitrage/opts.py @@ -29,6 +29,7 @@ import vitrage.notifier import vitrage.notifier.plugins.snmp import vitrage.os_clients import vitrage.rpc +import vitrage.storage LOG = log.getLogger(__name__) @@ -44,6 +45,7 @@ def list_opts(): ('datasources', vitrage.datasources.OPTS), ('evaluator', vitrage.evaluator.OPTS), ('consistency', vitrage.entity_graph.consistency.OPTS), + ('database', vitrage.storage.OPTS), ('entity_graph', vitrage.entity_graph.OPTS), ('service_credentials', vitrage.keystone_client.OPTS), ('machine_learning', diff --git a/vitrage/service.py b/vitrage/service.py index 5249ae5b1..e44dee48c 100644 --- a/vitrage/service.py +++ b/vitrage/service.py @@ -12,6 +12,7 @@ # under the License. from oslo_config import cfg +from oslo_db import options as db_options from oslo_log import log from oslo_policy import opts as policy_opts from osprofiler import initializer as osprofiler_initializer @@ -31,6 +32,7 @@ def prepare_service(args=None, conf=None, config_files=None): log.register_options(conf) policy_opts.set_defaults(conf) osprofiler_opts.set_defaults(conf) + db_options.set_defaults(conf) for group, options in opts.list_opts(): conf.register_opts(list(options), diff --git a/vitrage/storage/__init__.py b/vitrage/storage/__init__.py new file mode 100644 index 000000000..c9a7d8947 --- /dev/null +++ b/vitrage/storage/__init__.py @@ -0,0 +1,45 @@ +# Copyright 2017 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log +import six.moves.urllib.parse as urlparse +from stevedore import driver +import tenacity + +_NAMESPACE = 'vitrage.storage' + + +LOG = log.getLogger(__name__) + + +OPTS = [] + + +def get_connection_from_config(conf): + retries = conf.database.max_retries + url = conf.database.connection + connection_scheme = urlparse.urlparse(url).scheme + LOG.debug('looking for %(name)r driver in %(namespace)r', + {'name': connection_scheme, 'namespace': _NAMESPACE}) + mgr = driver.DriverManager(_NAMESPACE, connection_scheme) + + @tenacity.retry( + wait=tenacity.wait_fixed(conf.database.retry_interval), + stop=tenacity.stop_after_attempt(retries if retries >= 0 else 5), + reraise=True) + def _get_connection(): + """Return an open connection to the database.""" + return mgr.driver(conf, url) + + return _get_connection() diff --git a/vitrage/storage/base.py b/vitrage/storage/base.py new file mode 100644 index 000000000..32fdbbdbd --- /dev/null +++ b/vitrage/storage/base.py @@ -0,0 +1,24 @@ +# Copyright 2017 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class Connection(object): + """Base class for storage system connections.""" + + def __init__(self, conf, url): + pass + + @staticmethod + def upgrade(nocreate=False): + raise NotImplementedError('upgrade not implemented') diff --git a/vitrage/storage/impl_sqlalchemy.py b/vitrage/storage/impl_sqlalchemy.py new file mode 100644 index 000000000..1f58bfc40 --- /dev/null +++ b/vitrage/storage/impl_sqlalchemy.py @@ -0,0 +1,65 @@ +# Copyright 2017 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from __future__ import absolute_import + +from oslo_db.sqlalchemy import session as db_session +from oslo_log import log +from sqlalchemy.engine import url as sqlalchemy_url + +from vitrage import storage +from vitrage.storage import base +from vitrage.storage.sqlalchemy import models + +LOG = log.getLogger(__name__) + + +class Connection(base.Connection): + + def __init__(self, conf, url): + options = dict(conf.database.items()) + # set retries to 0 , since reconnection is already implemented + # in storage.__init__.get_connection_from_config function + options['max_retries'] = 0 + # add vitrage opts to database group + for opt in storage.OPTS: + options.pop(opt.name, None) + self._engine_facade = db_session.EngineFacade(self._dress_url(url), + **options) + self.conf = conf + + @staticmethod + def _dress_url(url): + # If no explicit driver has been set, we default to pymysql + if url.startswith("mysql://"): + url = sqlalchemy_url.make_url(url) + url.drivername = "mysql+pymysql" + return str(url) + return url + + def upgrade(self, nocreate=False): + engine = self._engine_facade.get_engine() + engine.connect() + models.Base.metadata.create_all(engine, checkfirst=False) + # TODO(ihefetz) upgrade logic is missing + + def disconnect(self): + self._engine_facade.get_engine().dispose() + + def clear(self): + engine = self._engine_facade.get_engine() + for table in reversed(models.Base.metadata.sorted_tables): + engine.execute(table.delete()) + engine.dispose() diff --git a/vitrage/db/__init__.py b/vitrage/storage/sqlalchemy/__init__.py similarity index 94% rename from vitrage/db/__init__.py rename to vitrage/storage/sqlalchemy/__init__.py index 7a05fb1f4..bf9f61d74 100644 --- a/vitrage/db/__init__.py +++ b/vitrage/storage/sqlalchemy/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2015 - Alcatel-Lucent +# Copyright 2017 - Nokia # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain diff --git a/vitrage/storage/sqlalchemy/models.py b/vitrage/storage/sqlalchemy/models.py new file mode 100644 index 000000000..11ff6df67 --- /dev/null +++ b/vitrage/storage/sqlalchemy/models.py @@ -0,0 +1,39 @@ +# Copyright 2017 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_db.sqlalchemy import models + +import six +from sqlalchemy.ext.declarative import declarative_base + + +class VitrageBase(models.TimestampMixin, models.ModelBase): + """Base class for Vitrage Models.""" + __table_args__ = {'mysql_charset': "utf8", + 'mysql_engine': "InnoDB"} + __table_initialized__ = False + + def __setitem__(self, key, value): + setattr(self, key, value) + + def __getitem__(self, key): + return getattr(self, key) + + def update(self, values): + """Make the model object behave like a dict.""" + for k, v in six.iteritems(values): + setattr(self, k, v) + + +Base = declarative_base(cls=VitrageBase) diff --git a/vitrage/tests/functional/api/__init__.py b/vitrage/tests/functional/api/__init__.py index 2aa368b37..ba16fa551 100644 --- a/vitrage/tests/functional/api/__init__.py +++ b/vitrage/tests/functional/api/__init__.py @@ -58,6 +58,11 @@ class FunctionalTest(base.BaseTest): group='api') self.CONF.set_override('auth_mode', self.auth, group='api') + + self.CONF.set_override('connection', + 'sqlite:///:memory:', + group='database') + self.app = webtest.TestApp(app.load_app(self.CONF)) def put_json(self, path, params, expect_errors=False, headers=None,