diff --git a/doc/source/running_tests.rst b/doc/source/running_tests.rst index 8bd01fc99..4b31c214b 100644 --- a/doc/source/running_tests.rst +++ b/doc/source/running_tests.rst @@ -153,6 +153,15 @@ For example, you want to run functional tests with keystone authentication enabled, input a valid set of credentials to ``[auth]`` section in configuration file and set ``auth_on`` parameter to ``True``. +Using local Mysql database +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To use a similar testing environment with database support like upstream CI, +you can run ``zaqar/tools/test-setup.sh`` to create a required Mysql user +``openstack_citest`` with same password. The user is required by oslo.db's +test. Zaqar needs it because Zaqar's sqlalchemy database migration is +leveraging oslo.db's migration test base. + .. rubric:: Footnotes .. [#f1] See http://docs.openstack.org/infra/system-config/jenkins.html diff --git a/test-requirements.txt b/test-requirements.txt index 60976c062..92945026b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -11,6 +11,7 @@ mock>=2.0 # BSD redis>=2.10.0 # MIT pymongo!=3.1,>=3.0.2 # Apache-2.0 websocket-client>=0.32.0 # LGPLv2+ +PyMySQL>=0.7.6 # MIT License # Unit testing coverage>=4.0 # Apache-2.0 diff --git a/tools/test-setup.sh b/tools/test-setup.sh new file mode 100755 index 000000000..2bf33a742 --- /dev/null +++ b/tools/test-setup.sh @@ -0,0 +1,33 @@ +#!/bin/bash -xe + +# This script will be run by OpenStack CI before unit tests are run, +# it sets up the test system as needed. +# Developers should setup their test systems in a similar way. + +# This setup needs to be run as a user that can run sudo. + +# The root password for the MySQL database; pass it in via +# MYSQL_ROOT_PW. +DB_ROOT_PW=${MYSQL_ROOT_PW:-insecure_slave} + +# This user and its password are used by the tests, if you change it, +# your tests might fail. +DB_USER=openstack_citest +DB_PW=openstack_citest + +sudo -H mysqladmin -u root password $DB_ROOT_PW + +# It's best practice to remove anonymous users from the database. If +# a anonymous user exists, then it matches first for connections and +# other connections from that host will not work. +sudo -H mysql -u root -p$DB_ROOT_PW -h localhost -e " + DELETE FROM mysql.user WHERE User=''; + FLUSH PRIVILEGES; + GRANT ALL PRIVILEGES ON *.* + TO '$DB_USER'@'%' identified by '$DB_PW' WITH GRANT OPTION;" + +# Now create our database. +mysql -u $DB_USER -p$DB_PW -h 127.0.0.1 -e " + SET default_storage_engine=MYISAM; + DROP DATABASE IF EXISTS openstack_citest; + CREATE DATABASE openstack_citest CHARACTER SET utf8;" \ No newline at end of file diff --git a/zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/001_liberty.py b/zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/001_liberty.py index 4f0d9e2f9..927d226a2 100644 --- a/zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/001_liberty.py +++ b/zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/001_liberty.py @@ -33,20 +33,20 @@ MYSQL_CHARSET = 'utf8' def upgrade(): - op.create_table('queues', + op.create_table('Queues', sa.Column('id', sa.INTEGER, primary_key=True), sa.Column('project', sa.String(64)), sa.Column('name', sa.String(64)), sa.Column('metadata', sa.LargeBinary), sa.UniqueConstraint('project', 'name')) - op.create_table('poolgroup', + op.create_table('PoolGroup', sa.Column('name', sa.String(64), primary_key=True)) - op.create_table('pools', + op.create_table('Pools', sa.Column('name', sa.String(64), primary_key=True), sa.Column('group', sa.String(64), - sa.ForeignKey('poolgroup.name', + sa.ForeignKey('PoolGroup.name', ondelete='CASCADE'), nullable=True), sa.Column('uri', sa.String(255), @@ -54,18 +54,18 @@ def upgrade(): sa.Column('weight', sa.INTEGER, nullable=False), sa.Column('options', sa.Text())) - op.create_table('flavors', + op.create_table('Flavors', sa.Column('name', sa.String(64), primary_key=True), sa.Column('project', sa.String(64)), sa.Column('pool_group', sa.String(64), - sa.ForeignKey('poolgroup.name', + sa.ForeignKey('PoolGroup.name', ondelete='CASCADE'), nullable=False), sa.Column('capabilities', sa.Text())) - op.create_table('catalogue', + op.create_table('Catalogue', sa.Column('pool', sa.String(64), - sa.ForeignKey('pools.name', + sa.ForeignKey('Pools.name', ondelete='CASCADE')), sa.Column('project', sa.String(64)), sa.Column('queue', sa.String(64), nullable=False), diff --git a/zaqar/tests/unit/storage/sqlalchemy_migration/test_migrations.py b/zaqar/tests/unit/storage/sqlalchemy_migration/test_migrations.py index 37a6916b3..aeccc3bf9 100644 --- a/zaqar/tests/unit/storage/sqlalchemy_migration/test_migrations.py +++ b/zaqar/tests/unit/storage/sqlalchemy_migration/test_migrations.py @@ -30,8 +30,6 @@ postgres=# create database openstack_citest with owner openstack_citest; """ -import os - from oslo_db.sqlalchemy import test_base from oslo_db.sqlalchemy import utils as db_utils @@ -95,17 +93,17 @@ class ZaqarMigrationsCheckers(object): 'metadata' ] self.assertColumnsExist( - engine, 'queues', queues_columns) + engine, 'Queues', queues_columns) self.assertColumnCount( - engine, 'queues', queues_columns) + engine, 'Queues', queues_columns) poolgroup_columns = [ 'name', ] self.assertColumnsExist( - engine, 'poolgroup', poolgroup_columns) + engine, 'PoolGroup', poolgroup_columns) self.assertColumnCount( - engine, 'poolgroup', poolgroup_columns) + engine, 'PoolGroup', poolgroup_columns) pools_columns = [ 'name', @@ -115,9 +113,9 @@ class ZaqarMigrationsCheckers(object): 'options', ] self.assertColumnsExist( - engine, 'pools', pools_columns) + engine, 'Pools', pools_columns) self.assertColumnCount( - engine, 'pools', pools_columns) + engine, 'Pools', pools_columns) flavors_columns = [ 'name', @@ -126,9 +124,9 @@ class ZaqarMigrationsCheckers(object): 'capabilities', ] self.assertColumnsExist( - engine, 'flavors', flavors_columns) + engine, 'Flavors', flavors_columns) self.assertColumnCount( - engine, 'flavors', flavors_columns) + engine, 'Flavors', flavors_columns) catalogue_columns = [ 'pool', @@ -136,19 +134,19 @@ class ZaqarMigrationsCheckers(object): 'queue', ] self.assertColumnsExist( - engine, 'catalogue', catalogue_columns) + engine, 'Catalogue', catalogue_columns) self.assertColumnCount( - engine, 'catalogue', catalogue_columns) + engine, 'Catalogue', catalogue_columns) self._data_001(engine, data) def _data_001(self, engine, data): - datasize = 512 * 1024 # 512kB - data = os.urandom(datasize) - t = db_utils.get_table(engine, 'job_binary_internal') - engine.execute(t.insert(), data=data, id='123', name='name') - new_data = engine.execute(t.select()).fetchone().data - self.assertEqual(data, new_data) + project = 'myproject' + t = db_utils.get_table(engine, 'Queues') + engine.execute(t.insert(), id='123', name='name', project='myproject', + metadata={}) + new_project = engine.execute(t.select()).fetchone().project + self.assertEqual(project, new_project) engine.execute(t.delete()) def _check_002(self, engine, data): diff --git a/zaqar/tests/unit/storage/sqlalchemy_migration/test_migrations_base.py b/zaqar/tests/unit/storage/sqlalchemy_migration/test_migrations_base.py index 2bba5f6fb..46fb48f8a 100644 --- a/zaqar/tests/unit/storage/sqlalchemy_migration/test_migrations_base.py +++ b/zaqar/tests/unit/storage/sqlalchemy_migration/test_migrations_base.py @@ -25,7 +25,6 @@ import io import os -import sqlalchemy as sa import alembic from alembic import command @@ -40,10 +39,17 @@ from zaqar.i18n import _LE import zaqar.storage.sqlalchemy.migration from zaqar.storage.sqlalchemy import tables - LOG = logging.getLogger(__name__) CONF = cfg.CONF +sqlalchemy_opts = [cfg.StrOpt('uri', + help='The SQLAlchemy connection string to' + ' use to connect to the database.', + secret=True)] + +CONF.register_opts(sqlalchemy_opts, + group='drivers:management_store:sqlalchemy') + class BaseWalkMigrationTestCase(object): @@ -63,10 +69,10 @@ class BaseWalkMigrationTestCase(object): should use oslo_config and openstack.commom.db.sqlalchemy.session with database functionality (reset default settings and session cleanup). """ + CONF.set_override('uri', str(engine.url), group='drivers:management_store:sqlalchemy', enforce_type=True) - sa.cleanup() def _alembic_command(self, alembic_command, engine, *args, **kwargs): """Most of alembic command return data into output. @@ -77,12 +83,12 @@ class BaseWalkMigrationTestCase(object): CONF.set_override('uri', str(engine.url), group='drivers:management_store:sqlalchemy', enforce_type=True) - sa.cleanup() + getattr(command, alembic_command)(*args, **kwargs) res = buf.getvalue().strip() LOG.debug('Alembic command {command} returns: {result}'.format( command=alembic_command, result=res)) - sa.cleanup() + return res def _get_versions(self): @@ -167,11 +173,9 @@ class TestModelsMigrationsSync(t_m.ModelsMigrationsSync): Allows to check if the DB schema obtained by applying of migration scripts is equal to the one produced from models definitions. """ - + mg_path = os.path.dirname(zaqar.storage.sqlalchemy.migration.__file__) ALEMBIC_CONFIG = alembic_config.Config( - os.path.join( - os.path.dirname(zaqar.storage.sqlalchemy.migration.__file__), - 'alembic.ini') + os.path.join(mg_path, 'alembic.ini') ) ALEMBIC_CONFIG.zaqar_config = CONF @@ -182,6 +186,8 @@ class TestModelsMigrationsSync(t_m.ModelsMigrationsSync): CONF.set_override('uri', str(engine.url), group='drivers:management_store:sqlalchemy', enforce_type=True) + script_location = os.path.join(self.mg_path, 'alembic_migrations') + self.ALEMBIC_CONFIG.set_main_option('script_location', script_location) alembic.command.upgrade(self.ALEMBIC_CONFIG, 'head') def get_metadata(self):