From 8164828f7b3b7b940f0a47148af98521bd1b0b8b Mon Sep 17 00:00:00 2001 From: jiasirui Date: Wed, 7 Dec 2022 09:37:47 +0800 Subject: [PATCH] add code of venusdatebase Change-Id: Id41d76baf1252f8f667b640fcde59e67d15f8a9f --- openstack_venus.egg-info/PKG-INFO | 60 +++++ openstack_venus.egg-info/SOURCES.txt | 254 ++++++++++++++++++ openstack_venus.egg-info/dependency_links.txt | 1 + openstack_venus.egg-info/entry_points.txt | 7 + openstack_venus.egg-info/not-zip-safe | 1 + openstack_venus.egg-info/requires.txt | 37 +++ openstack_venus.egg-info/top_level.txt | 1 + requirements.txt | 1 + setup.cfg | 3 + venus/cmd/manage.py | 38 +++ venus/conf/__init__.py | 2 + venus/conf/db.py | 25 ++ venus/db/__init__.py | 18 ++ venus/db/base.py | 39 +++ venus/db/migration.py | 58 ++++ venus/db/sqlalchemy/__init__.py | 0 venus/db/sqlalchemy/api.py | 161 +++++++++++ venus/db/sqlalchemy/migrate_repo/README | 4 + venus/db/sqlalchemy/migrate_repo/__init__.py | 0 venus/db/sqlalchemy/migrate_repo/manage.py | 24 ++ venus/db/sqlalchemy/migrate_repo/migrate.cfg | 20 ++ .../migrate_repo/versions/001_venus_init.py | 85 ++++++ .../migrate_repo/versions/__init__.py | 0 23 files changed, 839 insertions(+) create mode 100644 openstack_venus.egg-info/PKG-INFO create mode 100644 openstack_venus.egg-info/SOURCES.txt create mode 100644 openstack_venus.egg-info/dependency_links.txt create mode 100644 openstack_venus.egg-info/entry_points.txt create mode 100644 openstack_venus.egg-info/not-zip-safe create mode 100644 openstack_venus.egg-info/requires.txt create mode 100644 openstack_venus.egg-info/top_level.txt create mode 100644 venus/conf/db.py create mode 100644 venus/db/__init__.py create mode 100644 venus/db/base.py create mode 100644 venus/db/migration.py create mode 100644 venus/db/sqlalchemy/__init__.py create mode 100644 venus/db/sqlalchemy/api.py create mode 100644 venus/db/sqlalchemy/migrate_repo/README create mode 100644 venus/db/sqlalchemy/migrate_repo/__init__.py create mode 100644 venus/db/sqlalchemy/migrate_repo/manage.py create mode 100644 venus/db/sqlalchemy/migrate_repo/migrate.cfg create mode 100644 venus/db/sqlalchemy/migrate_repo/versions/001_venus_init.py create mode 100644 venus/db/sqlalchemy/migrate_repo/versions/__init__.py diff --git a/openstack_venus.egg-info/PKG-INFO b/openstack_venus.egg-info/PKG-INFO new file mode 100644 index 0000000..28c6cef --- /dev/null +++ b/openstack_venus.egg-info/PKG-INFO @@ -0,0 +1,60 @@ +Metadata-Version: 2.1 +Name: openstack-venus +Version: 1.1.0.dev26 +Summary: OpenStack Log Management as a Service +Home-page: UNKNOWN +Author: Brin Zhang +Author-email: zhangbailin@inspur.com +License: UNKNOWN +Description: Hello Venus + =========== + + An OpenStack Log Management Service. + + About Venus + ----------- + + In light of the problems and needs of retrieval, storage and analysis etc. + of logs on the OpenStack platform, we developed the OpenStack log management + module Venus. + + This project can provide a one-stop solution to log collection, + cleaning, indexing, analysis, alarm, visualization, report generation and + other needs, which involves helping operator or maintainer to quickly solve + retrieve problems, grasp the operational health of the platform, and improve + the level of platform management. + + Additionally, this project plans to use machine learning algorithms to + quickly locate IT failures and root causes, and improve operation and + maintenance efficiency. + + * Free software: Apache license + * Wiki: https://wiki.openstack.org/wiki/Venus + * Source: https://opendev.org/openstack/venus + * Blueprints: https://blueprints.launchpad.net/openstack-venus + * Bugs: https://bugs.launchpad.net/openstack-venus + * Documentation: https://docs.openstack.org/venus/latest/ + * Release notes: https://docs.openstack.org/releasenotes/venus/ + * Design specifications: https://specs.openstack.org/openstack/venus-specs/ + + Features + -------- + + * REST API for retrieval, storage and analysis of logs on the OpenStack platform. + * One-stop solution to log collection, cleaning, indexing, analysis, alarm, visualization, report generation. + + +Platform: UNKNOWN +Classifier: Environment :: OpenStack +Classifier: Intended Audience :: Information Technology +Classifier: Intended Audience :: System Administrators +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Operating System :: POSIX :: Linux +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Requires-Python: >=3.8 +Description-Content-Type: text/markdown diff --git a/openstack_venus.egg-info/SOURCES.txt b/openstack_venus.egg-info/SOURCES.txt new file mode 100644 index 0000000..a26a428 --- /dev/null +++ b/openstack_venus.egg-info/SOURCES.txt @@ -0,0 +1,254 @@ +.stestr.conf +.zuul.yaml +AUTHORS +CONTRIBUTING.rst +ChangeLog +HACKING.rst +LICENSE +MANIFEST.in +README.bak.rst +README.rst +requirements.txt +setup.cfg +setup.py +test-requirements.txt +tox.ini +api-ref/source/conf.py +api-ref/source/index.rst +api-ref/source/status.yaml +api-ref/source/v1/api-versions.inc +api-ref/source/v1/custom-config.inc +api-ref/source/v1/index.rst +api-ref/source/v1/parameters.yaml +api-ref/source/v1/search-analyse-logs.inc +api-ref/source/v1/search-instance-callchain.inc +api-ref/source/v1/search-instance-requestids.inc +api-ref/source/v1/search-logs-download.inc +api-ref/source/v1/search-logs.inc +api-ref/source/v1/search-params.inc +api-ref/source/v1/search-typical-logs.inc +api-ref/source/v1/samples/versions-response.json +api-ref/source/v1/samples/custom-config/custom-config-show-response.json +api-ref/source/v1/samples/search-log/search-analyse-logs-show-response.json +api-ref/source/v1/samples/search-log/search-instance-callchain-show-response.json +api-ref/source/v1/samples/search-log/search-instance-request-ids-show-response.json +api-ref/source/v1/samples/search-log/search-logs-download-show-response.json +api-ref/source/v1/samples/search-log/search-logs-show-response.json +api-ref/source/v1/samples/search-log/search-params-show-response.json +api-ref/source/v1/samples/search-log/search-typical-logs-show-response.json +devstack/README.md +devstack/plugin.sh +devstack/settings +devstack/fluentd-conf/td-agent.conf +devstack/fluentd-conf/filter/00-record_transformer.conf +devstack/fluentd-conf/filter/01-rewrite.conf.c +devstack/fluentd-conf/format/apache_access.conf +devstack/fluentd-conf/format/wsgi_access.conf +devstack/fluentd-conf/input/barbican.conf +devstack/fluentd-conf/input/cinder.conf +devstack/fluentd-conf/input/cyborg.conf +devstack/fluentd-conf/input/designate.conf +devstack/fluentd-conf/input/glance.conf +devstack/fluentd-conf/input/heat.conf +devstack/fluentd-conf/input/ironic.conf +devstack/fluentd-conf/input/keystone.conf +devstack/fluentd-conf/input/magnum.conf +devstack/fluentd-conf/input/manila.conf +devstack/fluentd-conf/input/neutron.conf +devstack/fluentd-conf/input/nova.conf +devstack/fluentd-conf/input/octavia.conf +devstack/fluentd-conf/input/palcement.conf +devstack/fluentd-conf/input/senlin.conf +devstack/fluentd-conf/input/swift.conf +devstack/fluentd-conf/input/syslog.conf +devstack/fluentd-conf/input/venus.conf +devstack/fluentd-conf/input/vitrage.conf +devstack/fluentd-conf/input/zun.conf +devstack/fluentd-conf/output/00-local.conf +devstack/fluentd-conf/output/01-es.conf +doc/README.rst +doc/requirements.txt +doc/source/conf.py +doc/source/index.rst +doc/source/contributor/contributing.rst +doc/source/contributor/index.rst +doc/source/contributor/releasenotes.rst +etc/venus/README-venus.conf.sample +etc/venus/api-httpd.conf +etc/venus/api-paste.ini +etc/venus/logging_sample.conf +etc/venus/policy.json +etc/venus/rootwrap.conf +etc/venus/venus.conf +etc/venus/rootwrap.d/venus.filters +openstack_venus.egg-info/PKG-INFO +openstack_venus.egg-info/SOURCES.txt +openstack_venus.egg-info/dependency_links.txt +openstack_venus.egg-info/entry_points.txt +openstack_venus.egg-info/not-zip-safe +openstack_venus.egg-info/requires.txt +openstack_venus.egg-info/top_level.txt +releasenotes/notes/.placeholder +releasenotes/source/conf.py +releasenotes/source/index.rst +releasenotes/source/unreleased.rst +releasenotes/source/zed.rst +tools/flake8wrap.sh +tools/config/venus-config-generator.conf +tools/config/venus-policy-generator.conf +venus/__init__.py +venus/context.py +venus/exception.py +venus/i18n.py +venus/service.py +venus/utils.py +venus/version.py +venus/api/__init__.py +venus/api/common.py +venus/api/extensions.py +venus/api/urlmap.py +venus/api/xmlutil.py +venus/api/contrib/__init__.py +venus/api/middleware/__init__.py +venus/api/middleware/auth.py +venus/api/middleware/env.py +venus/api/middleware/fault.py +venus/api/middleware/sizelimit.py +venus/api/openstack/__init__.py +venus/api/openstack/wsgi.py +venus/api/schemas/atom-link.rng +venus/api/schemas/v1.1/extension.rng +venus/api/schemas/v1.1/extensions.rng +venus/api/v1/__init__.py +venus/api/v1/router.py +venus/api/v1/views/__init__.py +venus/api/v1/views/versions.py +venus/cmd/__init__.py +venus/cmd/api.py +venus/cmd/manage.py +venus/common/__init__.py +venus/common/utils.py +venus/conf/__init__.py +venus/conf/api.py +venus/conf/common.py +venus/conf/core.py +venus/conf/db.py +venus/conf/elasticsearch.py +venus/conf/profiler.py +venus/db/__init__.py +venus/hacking/__init__.py +venus/hacking/checks.py +venus/locale/venus-log-info.pot +venus/locale/venus-log-warning.pot +venus/locale/venus.pot +venus/modules/__init__.py +venus/modules/custom_config/__init__.py +venus/modules/custom_config/controller.py +venus/modules/search/__init__.py +venus/modules/search/action.py +venus/modules/search/controller.py +venus/modules/search/es_template.py +venus/modules/search/search_lib.py +venus/modules/version/__init__.py +venus/modules/version/versions.py +venus/objects/__init__.py +venus/objects/base.py +venus/openstack/__init__.py +venus/openstack/common/README +venus/openstack/common/__init__.py +venus/openstack/common/_i18n.py +venus/openstack/common/gettextutils.py +venus/openstack/common/config/__init__.py +venus/openstack/common/config/generator.py +venus/task/__init__.py +venus/task/adapter.py +venus/task/timer.py +venus/task/core/__init__.py +venus/task/core/delete_es_index_task.py +venus/tests/__init__.py +venus/tests/test_hacking.py +venus/tests/functional/__init__.py +venus/tests/unit/__init__.py +venus/tests/unit/fakes.py +venus/tests/unit/api/__init__.py +venus/tests/unit/api/test_search_action.py +venus/tests/unit/api/test_search_controller.py +venus/wsgi/__init__.py +venus/wsgi/common.py +venus/wsgi/eventlet_server.py +venus/wsgi/wsgi.py +venus/api/__init__.py +venus/api/common.py +venus/api/extensions.py +venus/api/urlmap.py +venus/api/xmlutil.py +venus/api/contrib/__init__.py +venus/api/middleware/__init__.py +venus/api/middleware/auth.py +venus/api/middleware/env.py +venus/api/middleware/fault.py +venus/api/middleware/sizelimit.py +venus/api/openstack/__init__.py +venus/api/openstack/wsgi.py +venus/api/v1/__init__.py +venus/api/v1/router.py +venus/api/v1/views/__init__.py +venus/api/v1/views/versions.py +venus/cmd/__init__.py +venus/cmd/api.py +venus/cmd/manage.py +venus/common/__init__.py +venus/common/utils.py +venus/conf/__init__.py +venus/conf/api.py +venus/conf/common.py +venus/conf/core.py +venus/conf/db.py +venus/conf/elasticsearch.py +venus/conf/profiler.py +venus/db/__init__.py +venus/db/base.py +venus/db/migration.py +venus/db/sqlalchemy/__init__.py +venus/db/sqlalchemy/api.py +venus/db/sqlalchemy/migrate_repo/__init__.py +venus/db/sqlalchemy/migrate_repo/manage.py +venus/db/sqlalchemy/migrate_repo/versions/001_venus_init.py +venus/db/sqlalchemy/migrate_repo/versions/__init__.py +venus/hacking/__init__.py +venus/hacking/checks.py +venus/modules/__init__.py +venus/modules/custom_config/__init__.py +venus/modules/custom_config/controller.py +venus/modules/search/__init__.py +venus/modules/search/action.py +venus/modules/search/controller.py +venus/modules/search/es_template.py +venus/modules/search/search_lib.py +venus/modules/version/__init__.py +venus/modules/version/versions.py +venus/objects/__init__.py +venus/objects/base.py +venus/openstack/__init__.py +venus/openstack/common/__init__.py +venus/openstack/common/_i18n.py +venus/openstack/common/gettextutils.py +venus/openstack/common/config/__init__.py +venus/openstack/common/config/generator.py +venus/task/__init__.py +venus/task/adapter.py +venus/task/timer.py +venus/task/core/__init__.py +venus/task/core/delete_es_index_task.py +venus/tests/__init__.py +venus/tests/test_hacking.py +venus/tests/functional/__init__.py +venus/tests/unit/__init__.py +venus/tests/unit/fakes.py +venus/tests/unit/api/__init__.py +venus/tests/unit/api/test_search_action.py +venus/tests/unit/api/test_search_controller.py +venus/wsgi/__init__.py +venus/wsgi/common.py +venus/wsgi/eventlet_server.py +venus/wsgi/wsgi.py \ No newline at end of file diff --git a/openstack_venus.egg-info/dependency_links.txt b/openstack_venus.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/openstack_venus.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/openstack_venus.egg-info/entry_points.txt b/openstack_venus.egg-info/entry_points.txt new file mode 100644 index 0000000..63c0b83 --- /dev/null +++ b/openstack_venus.egg-info/entry_points.txt @@ -0,0 +1,7 @@ +[console_scripts] +venus-api = venus.cmd.api:main +venus-manager = venus.cmd.manage:main + +[venus.database.migration_backend] +sqlalchemy = oslo_db.sqlalchemy.migration + diff --git a/openstack_venus.egg-info/not-zip-safe b/openstack_venus.egg-info/not-zip-safe new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/openstack_venus.egg-info/not-zip-safe @@ -0,0 +1 @@ + diff --git a/openstack_venus.egg-info/requires.txt b/openstack_venus.egg-info/requires.txt new file mode 100644 index 0000000..dcf8bad --- /dev/null +++ b/openstack_venus.egg-info/requires.txt @@ -0,0 +1,37 @@ +pbr!=2.1.0,>=0.11 +pecan!=1.0.2,!=1.0.3,!=1.0.4,!=1.2,>=1.0.0 +WSME>=0.10.1 +eventlet>=0.26.0 +oslo.i18n>=1.5.0 +oslo.log>=5.0.0 +oslo.context>=2.9.0 +oslo.messaging>=10.3.0 +oslo.concurrency>=3.26.0 +oslo.service!=1.28.1,>=1.0.0 +oslo.db>=10.0.0 +os-resource-classes>=0.5.0 +oslo.upgradecheck>=0.1.0 +oslo.utils>=4.5.0 +oslo.versionedobjects>=1.31.2 +osprofiler>=3.4.0 +SQLAlchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8,>=0.9.0 +alembic>=0.8.10 +stevedore>=1.5.0 +keystonemiddleware>=4.17.0 +jsonpatch!=1.20,>=1.16 +psutil>=3.2.2 +python-glanceclient>=2.3.0 +cursive>=0.2.1 +microversion_parse>=0.2.1 +openstacksdk>=0.46.0 +elasticsearch<3.0.0,>=2.0.0 +apscheduler>=3.6.0 +retrying>=1.3.3 +lxml>=4.6.2 +PyMySQL>=0.7.11 + +[:(python_version=='3.6')] +oslo.log>=4.2.1 + +[:(python_version=='3.7')] +oslo.log>=4.2.1 diff --git a/openstack_venus.egg-info/top_level.txt b/openstack_venus.egg-info/top_level.txt new file mode 100644 index 0000000..acf16d8 --- /dev/null +++ b/openstack_venus.egg-info/top_level.txt @@ -0,0 +1 @@ +venus diff --git a/requirements.txt b/requirements.txt index 83305e7..bb17477 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,3 +34,4 @@ elasticsearch>=2.0.0,<3.0.0 # Apache-2.0 apscheduler>=3.6.0 # MIT License retrying>=1.3.3 lxml>=4.6.2 +PyMySQL>=0.7.11 diff --git a/setup.cfg b/setup.cfg index 416840c..f4be419 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,3 +28,6 @@ packages = console_scripts = venus-manager = venus.cmd.manage:main venus-api = venus.cmd.api:main + +venus.database.migration_backend = + sqlalchemy = oslo_db.sqlalchemy.migration diff --git a/venus/cmd/manage.py b/venus/cmd/manage.py index be6ffe9..3b7e63a 100644 --- a/venus/cmd/manage.py +++ b/venus/cmd/manage.py @@ -22,6 +22,7 @@ import os import sys from oslo_config import cfg +from oslo_db.sqlalchemy import migration from oslo_log import log as logging from venus.conf import CONF @@ -31,6 +32,12 @@ from venus import objects from venus.task import timer from venus import version +from venus import context +from venus import db +from venus.db import migration as db_migration +from venus.db.sqlalchemy import api as db_api + + i18n.enable_lazy() @@ -220,8 +227,39 @@ class TaskCommands(object): timer.init_advanced_timer() +class DbCommands(object): + """Class for managing the database.""" + + def __init__(self): + pass + + @args('version', nargs='?', default=None, + help='Database version') + def sync(self, version=None): + """Sync the database up to the most recent version.""" + return db_migration.db_sync(version) + + def version(self): + """Print the current database version.""" + print(migration.db_version(db_api.get_engine(), + db_migration.MIGRATE_REPO_PATH, + db_migration.INIT_VERSION)) + + @args('age_in_days', type=int, + help='Purge deleted rows older than age in days') + def purge(self, age_in_days): + """Purge deleted rows older than a given age from venus tables.""" + age_in_days = int(age_in_days) + if age_in_days <= 0: + print(_("Must supply a positive, non-zero value for age")) + exit(1) + ctxt = context.get_admin_context() + db.purge_deleted_rows(ctxt, age_in_days) + + CATEGORIES = { 'config': ConfigCommands, + 'db': DbCommands, 'logs': GetLogCommands, 'shell': ShellCommands, 'version': VersionCommands, diff --git a/venus/conf/__init__.py b/venus/conf/__init__.py index d6322d4..2b8c6b9 100644 --- a/venus/conf/__init__.py +++ b/venus/conf/__init__.py @@ -17,6 +17,7 @@ from oslo_config import cfg from venus.conf import api from venus.conf import common from venus.conf import core +from venus.conf import db from venus.conf import elasticsearch from venus.conf import profiler @@ -27,3 +28,4 @@ common.register_opts(CONF) core.register_opts(CONF) elasticsearch.register_opts(CONF) profiler.register_opts(CONF) +db.register_opts(CONF) diff --git a/venus/conf/db.py b/venus/conf/db.py new file mode 100644 index 0000000..bc62614 --- /dev/null +++ b/venus/conf/db.py @@ -0,0 +1,25 @@ +# Copyright 2020 Inspur +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg +from oslo_db import options + +db_driver_opt = cfg.StrOpt('db_driver', + default='venus.db', + help='Driver to use for database access') + + +def register_opts(conf): + options.set_defaults(conf, connection='sqlite:///$state_path/venus.sqlite') + conf.register_opt(db_driver_opt) diff --git a/venus/db/__init__.py b/venus/db/__init__.py new file mode 100644 index 0000000..4ca34c8 --- /dev/null +++ b/venus/db/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +DB abstraction for Venus +""" diff --git a/venus/db/base.py b/venus/db/base.py new file mode 100644 index 0000000..eed38c9 --- /dev/null +++ b/venus/db/base.py @@ -0,0 +1,39 @@ +# Copyright 2020 Inspur +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Base class for classes that need modular database access.""" + +from oslo_config import cfg +from oslo_utils import importutils + + +db_driver_opt = cfg.StrOpt('db_driver', + default='venus.db', + help='Driver to use for database access') + +CONF = cfg.CONF +CONF.register_opt(db_driver_opt) + + +class Base(object): + """DB driver is injected in the init method.""" + + def __init__(self, db_driver=None): + # NOTE(mriedem): Without this call, multiple inheritance involving + # the db Base class does not work correctly. + super(Base, self).__init__() + if not db_driver: + db_driver = CONF.db_driver + self.db = importutils.import_module(db_driver) # pylint: disable=C0103 + self.db.dispose_engine() diff --git a/venus/db/migration.py b/venus/db/migration.py new file mode 100644 index 0000000..f446c52 --- /dev/null +++ b/venus/db/migration.py @@ -0,0 +1,58 @@ +# Copyright 2020 Inspur +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Database setup and migration commands.""" + +import os +import threading + +from oslo_config import cfg +from oslo_db import options +from stevedore import driver + +from venus.db.sqlalchemy import api as db_api + +INIT_VERSION = 000 + +_IMPL = None +_LOCK = threading.Lock() + +options.set_defaults(cfg.CONF) + +MIGRATE_REPO_PATH = os.path.join( + os.path.abspath(os.path.dirname(__file__)), + 'sqlalchemy', + 'migrate_repo', +) + + +def get_backend(): + global _IMPL + if _IMPL is None: + with _LOCK: + if _IMPL is None: + _IMPL = driver.DriverManager( + "venus.database.migration_backend", + cfg.CONF.database.backend).driver + return _IMPL + + +def db_sync(version=None, init_version=INIT_VERSION, engine=None): + """Migrate the database to `version` or the most recent version.""" + if engine is None: + engine = db_api.get_engine() + return get_backend().db_sync(engine=engine, + abs_path=MIGRATE_REPO_PATH, + version=version, + init_version=init_version) diff --git a/venus/db/sqlalchemy/__init__.py b/venus/db/sqlalchemy/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/venus/db/sqlalchemy/api.py b/venus/db/sqlalchemy/api.py new file mode 100644 index 0000000..95d9158 --- /dev/null +++ b/venus/db/sqlalchemy/api.py @@ -0,0 +1,161 @@ +# Copyright 2020 Inspur +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Implementation of SQLAlchemy backend.""" + +import sqlalchemy +import sys +import threading +import warnings + +from oslo_config import cfg +from oslo_db import options +from oslo_db.sqlalchemy import session as db_session +from oslo_log import log as logging +import osprofiler.sqlalchemy + +from venus import exception +from venus.i18n import _ + +CONF = cfg.CONF +CONF.import_group("profiler", "venus.service") +log = logging.getLogger(__name__) + +options.set_defaults(CONF, connection='sqlite:///$state_path/venus.sqlite') + +_LOCK = threading.Lock() +_FACADE = None + + +def _create_facade_lazily(): + global _LOCK + with _LOCK: + global _FACADE + if _FACADE is None: + _FACADE = db_session.EngineFacade( + CONF.database.connection, + **dict(CONF.database) + ) + + if CONF.profiler.profiler_enabled: + if CONF.profiler.trace_sqlalchemy: + osprofiler.sqlalchemy.add_tracing(sqlalchemy, + _FACADE.get_engine(), + "db") + + return _FACADE + + +def get_engine(): + facade = _create_facade_lazily() + return facade.get_engine() + + +def get_session(**kwargs): + facade = _create_facade_lazily() + return facade.get_session(**kwargs) + + +def dispose_engine(): + get_engine().dispose() + + +_DEFAULT_QUOTA_NAME = 'default' + + +def get_backend(): + """The backend is this module itself.""" + + return sys.modules[__name__] + + +def is_admin_context(context): + """Indicates if the request context is an administrator.""" + if not context: + warnings.warn(_('Use of empty request context is deprecated'), + DeprecationWarning) + raise Exception('die') + return context.is_admin + + +def is_user_context(context): + """Indicates if the request context is a normal user.""" + if not context: + return False + if context.is_admin: + return False + if not context.user_id or not context.project_id: + return False + return True + + +def authorize_project_context(context, project_id): + """Ensures a request has permission to access the given project.""" + if is_user_context(context): + if not context.project_id: + raise exception.NotAuthorized() + elif context.project_id != project_id: + raise exception.NotAuthorized() + + +def authorize_user_context(context, user_id): + """Ensures a request has permission to access the given user.""" + if is_user_context(context): + if not context.user_id: + raise exception.NotAuthorized() + elif context.user_id != user_id: + raise exception.NotAuthorized() + + +def authorize_quota_class_context(context, class_name): + """Ensures a request has permission to access the given quota class.""" + if is_user_context(context): + if not context.quota_class: + raise exception.NotAuthorized() + elif context.quota_class != class_name: + raise exception.NotAuthorized() + + +def require_admin_context(f): + """Decorator to require admin request context. + + The first argument to the wrapped function must be the context. + + """ + + def wrapper(*args, **kwargs): + if not is_admin_context(args[0]): + raise exception.AdminRequired() + return f(*args, **kwargs) + + return wrapper + + +def require_context(f): + """Decorator to require *any* user or admin context. + + This does no authorization for user or project access matching, see + :py:func:`authorize_project_context` and + :py:func:`authorize_user_context`. + + The first argument to the wrapped function must be the context. + + """ + + def wrapper(*args, **kwargs): + if not is_admin_context(args[0]) and not is_user_context(args[0]): + raise exception.NotAuthorized() + return f(*args, **kwargs) + + return wrapper diff --git a/venus/db/sqlalchemy/migrate_repo/README b/venus/db/sqlalchemy/migrate_repo/README new file mode 100644 index 0000000..6218f8c --- /dev/null +++ b/venus/db/sqlalchemy/migrate_repo/README @@ -0,0 +1,4 @@ +This is a database migration repository. + +More information at +http://code.google.com/p/sqlalchemy-migrate/ diff --git a/venus/db/sqlalchemy/migrate_repo/__init__.py b/venus/db/sqlalchemy/migrate_repo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/venus/db/sqlalchemy/migrate_repo/manage.py b/venus/db/sqlalchemy/migrate_repo/manage.py new file mode 100644 index 0000000..01e7193 --- /dev/null +++ b/venus/db/sqlalchemy/migrate_repo/manage.py @@ -0,0 +1,24 @@ +# Copyright 2020 Inspur +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +from migrate.versioning.shell import main + +from venus.db.sqlalchemy import migrate_repo + + +if __name__ == '__main__': + main(debug='False', + repository=os.path.abspath(os.path.dirname(migrate_repo.__file__))) diff --git a/venus/db/sqlalchemy/migrate_repo/migrate.cfg b/venus/db/sqlalchemy/migrate_repo/migrate.cfg new file mode 100644 index 0000000..0734850 --- /dev/null +++ b/venus/db/sqlalchemy/migrate_repo/migrate.cfg @@ -0,0 +1,20 @@ +[db_settings] +# Used to identify which repository this database is versioned under. +# You can use the name of your project. +repository_id=venus + +# The name of the database table used to track the schema version. +# This name shouldn't already be used by your project. +# If this is changed once a database is under version control, you'll need to +# change the table name in each database too. +version_table=migrate_version + +# When committing a change script, Migrate will attempt to generate the +# sql for all supported databases; normally, if one of them fails - probably +# because you don't have that database installed - it is ignored and the +# commit continues, perhaps ending successfully. +# Databases in this list MUST compile successfully during a commit, or the +# entire commit will fail. List the databases your application will actually +# be using to ensure your updates to that database work properly. +# This must be a list; example: ['postgres','sqlite'] +required_dbs=[] diff --git a/venus/db/sqlalchemy/migrate_repo/versions/001_venus_init.py b/venus/db/sqlalchemy/migrate_repo/versions/001_venus_init.py new file mode 100644 index 0000000..b05b982 --- /dev/null +++ b/venus/db/sqlalchemy/migrate_repo/versions/001_venus_init.py @@ -0,0 +1,85 @@ +# Copyright 2020 Inspur +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""WSGI Routers for the Identity service.""" + +import sqlalchemy as sql +from sqlalchemy.orm import sessionmaker + + +def upgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + t_mo_regitster_task = sql.Table( + 't_mo_regitster_task', + meta, + sql.Column('Id', sql.Integer, nullable=False, + primary_key=True), + sql.Column('task_name', sql.String(255), nullable=True, + primary_key=False), + sql.Column('host_name', sql.String(255), nullable=True, + primary_key=False), + sql.Column('update_time', sql.DateTime, nullable=True, + primary_key=False, default='0000-00-00 00:00:00'), + sql.Column('created_at', sql.DateTime, + nullable=True, primary_key=False), + sql.Column('updated_at', sql.DateTime, + nullable=True, primary_key=False), + sql.Column('deleted', sql.String(1), + nullable=True, primary_key=False), + sql.Column('deleted_at', sql.DateTime, + nullable=True, primary_key=False), + + mysql_engine='InnoDB', + mysql_charset='utf8') + + t_mo_regitster_task.create(migrate_engine, checkfirst=True) + + new_data = { + 'Id': '1', + 'task_name': 'delete_es_index', + 'host_name': '', + 'update_time': '1900-01-01 00:00:00' + } + maker = sessionmaker(bind=migrate_engine) + session = maker() + t_mo_regitster_task = sql.Table('t_mo_regitster_task', meta, autoload=True) + row = t_mo_regitster_task.insert().values(**new_data) + session.execute(row) + session.commit() + + t_mo_custom_config = sql.Table( + 't_mo_custom_config', + meta, + sql.Column('id', sql.String(64), primary_key=True), + sql.Column('value', sql.String(10240), nullable=False), + sql.Column('update_time', sql.DateTime), + mysql_engine='InnoDB', + mysql_charset='utf8') + + t_mo_custom_config.create(migrate_engine, checkfirst=True) + new_data = { + 'id': 'es_index_length', + 'value': '30', + 'update_time': '1900-01-01 00:00:00' + } + maker = sessionmaker(bind=migrate_engine) + session = maker() + t_mo_custom_config = sql.Table('t_mo_custom_config', meta, autoload=True) + row = t_mo_custom_config.insert().values(**new_data) + session.execute(row) + + session.commit() + session.close() diff --git a/venus/db/sqlalchemy/migrate_repo/versions/__init__.py b/venus/db/sqlalchemy/migrate_repo/versions/__init__.py new file mode 100644 index 0000000..e69de29