From 8049697fc962c61d56f4e7ed1208fdd40dd3e0ec Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Thu, 15 Sep 2022 15:59:06 +0000 Subject: [PATCH] Switch to 2023.1 Python3 unit tests and generic template name This is an automatically generated patch to ensure unit testing is in place for all the of the tested runtimes for antelope. Also, updating the template name to generic one. See also the PTI in governance [1]. [1]: https://governance.openstack.org/tc/reference/project-testing-interface.html Change-Id: I29fcb9afa016b1321b0b4c0118a8d7ffbc00b02e --- .zuul.yaml | 8 ++------ bindep.txt | 1 - devstack/plugin.sh | 26 +++++++++++++++----------- requirements.txt | 16 ++++++++-------- tools/test-setup.sh | 19 +++++++++++++++++++ tox.ini | 16 ++++++++++------ zaqar/storage/mongodb/flavors.py | 15 ++++++++++----- zaqar/storage/mongodb/pools.py | 28 ++++++++++++++++++---------- zaqar/storage/utils.py | 15 ++++++++------- 9 files changed, 90 insertions(+), 54 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index 14ae1bec3..afc8cc95a 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -75,8 +75,8 @@ - project: templates: - - openstack-python3-zed-jobs - - openstack-python3-zed-jobs-arm64 + - openstack-python3-jobs + - openstack-python3-jobs-arm64 - publish-openstack-docs-pti - periodic-stable-jobs - check-requirements @@ -92,16 +92,12 @@ - zaqar-grenade-mongodb - zaqar-grenade-redis - zaqar-grenade-swift - # Heat job that deploy Zaqar - - heat-functional-convg-mysql-lbaasv2 - zaqar-rally: voting: false gate: - queue: zaqar jobs: - zaqar-tox-integration - zaqar-tempest-plugin-mongodb - zaqar-tempest-plugin-redis - zaqar-tempest-plugin-swift - zaqar-tempest-plugin-swift-ipv6 - - heat-functional-convg-mysql-lbaasv2 diff --git a/bindep.txt b/bindep.txt index 5afd238c5..0603c33ed 100644 --- a/bindep.txt +++ b/bindep.txt @@ -3,7 +3,6 @@ # their own bindep.txt files if the list below is not # working for them. -mongodb [platform:dpkg] mongodb-server [platform:rpm !platform:opensuseproject] dev-db/mongodb [platform:gentoo] redis [platform:rpm] diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 788a0ed15..dd81b92b8 100755 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -50,15 +50,16 @@ function cleanup_zaqar { # cleanup_zaqar_mongodb() - Remove residual data files, anything left over from previous # runs that a clean run would need to clean up +# After mongodb 6.0, the mongo shell has been remove, now using mongosh. function cleanup_zaqar_mongodb { - if ! timeout $SERVICE_TIMEOUT sh -c "while ! mongo zaqar --eval 'db.dropDatabase();'; do sleep 1; done"; then + if ! timeout $SERVICE_TIMEOUT sh -c "while ! mongosh zaqar --eval 'db.dropDatabase();'; do sleep 1; done"; then die $LINENO "Mongo DB did not start" else - full_version=$(mongo zaqar --eval 'db.dropDatabase();') - mongo_version=`echo $full_version | cut -d' ' -f4` - required_mongo_version='2.2' + full_version=$(mongosh zaqar --eval 'db.dropDatabase();') + mongo_version=`echo $full_version | cut -d' ' -f11` + required_mongo_version='6.0' if [[ $mongo_version < $required_mongo_version ]]; then - die $LINENO "Zaqar needs Mongo DB version >= 2.2 to run." + die $LINENO "Zaqar needs Mongo DB version >= 6.0 to run." fi fi } @@ -186,15 +187,18 @@ function configure_mongodb { # per database. pip_install pymongo if is_ubuntu; then - install_package mongodb-server - if ! grep -qF "smallfiles = true" /etc/mongodb.conf; then - echo "smallfiles = true" | sudo tee --append /etc/mongodb.conf > /dev/null - fi - restart_service mongodb + # NOTE: To fix the mongodb's issue in ubuntu 22.04 LTS + wget -qO - https://www.mongodb.org/static/pgp/server-6.0.asc | sudo apt-key add - + echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/6.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-6.0.list + sudo apt update + curl -LO http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1-1ubuntu2.1~18.04.20_amd64.deb + sudo dpkg -i ./libssl1.1_1.1.1-1ubuntu2.1~18.04.20_amd64.deb + + install_package mongodb-org + restart_service mongod elif is_fedora; then install_package mongodb install_package mongodb-server - sudo sed -i '/--smallfiles/!s/OPTIONS=\"/OPTIONS=\"--smallfiles /' /etc/sysconfig/mongod restart_service mongod fi } diff --git a/requirements.txt b/requirements.txt index 1d5b0092c..5efcdcacf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,17 +16,17 @@ python-swiftclient>=3.10.1 # Apache-2.0 WebOb>=1.7.1 # MIT stevedore>=3.2.2 # Apache-2.0 oslo.cache>=1.26.0 # Apache-2.0 -oslo.config>=6.8.0 # Apache-2.0 +oslo.config>=8.3.2 # Apache-2.0 oslo.context>=2.19.2 # Apache-2.0 -oslo.db>=8.4.0 # Apache-2.0 +oslo.db>=11.0.0 # Apache-2.0 oslo.i18n>=3.15.3 # Apache-2.0 -oslo.log>=3.36.0 # Apache-2.0 -oslo.messaging>=5.29.0 # Apache-2.0 -oslo.reports>=1.18.0 # Apache-2.0 -oslo.serialization!=2.19.1,>=2.18.0 # Apache-2.0 +oslo.log>=4.6.1 # Apache-2.0 +oslo.messaging>=12.5.0 # Apache-2.0 +oslo.reports>=2.2.0 # Apache-2.0 +oslo.serialization>=4.2.0 # Apache-2.0 oslo.upgradecheck>=1.3.0 # Apache-2.0 -oslo.utils>=3.33.0 # Apache-2.0 -oslo.policy>=3.6.0 # Apache-2.0 +oslo.utils>=4.12.1 # Apache-2.0 +oslo.policy>=3.8.1 # Apache-2.0 osprofiler>=1.4.0 # Apache-2.0 SQLAlchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8,>=1.3.19 # MIT autobahn>=22.3.2 # MIT License diff --git a/tools/test-setup.sh b/tools/test-setup.sh index 3cf57b233..9090c018b 100755 --- a/tools/test-setup.sh +++ b/tools/test-setup.sh @@ -31,3 +31,22 @@ mysql -u $DB_USER -p$DB_PW -h 127.0.0.1 -e " SET default_storage_engine=MYISAM; DROP DATABASE IF EXISTS openstack_citest; CREATE DATABASE openstack_citest CHARACTER SET utf8;" + +# TO fix the mongodb issue in ubuntu 22.04 +ubuntu_version=`cat /etc/issue | cut -d " " -f2` +if [[ $ubuntu_version > '22' ]]; then + wget -qO - https://www.mongodb.org/static/pgp/server-6.0.asc | sudo apt-key add - + echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/6.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-6.0.list + sudo apt update + curl -LO http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1-1ubuntu2.1~18.04.20_amd64.deb + sudo dpkg -i ./libssl1.1_1.1.1-1ubuntu2.1~18.04.20_amd64.deb + sudo apt install -y mongodb-org + sudo systemctl restart mongod +else + sudo apt-get install -y mongodb + sudo systemctl restart mongodb +fi + +sudo apt install -y pip +pip install setuptools +sudo python3 setup.py install diff --git a/tox.ini b/tox.ini index 9eaed719b..e55a69c21 100644 --- a/tox.ini +++ b/tox.ini @@ -17,13 +17,11 @@ setenv = VIRTUAL_ENV={envdir} OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 OS_TEST_TIMEOUT=60 -deps = -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} - -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt +deps = + -r{toxinidir}/test-requirements.txt + -r{toxinidir}/requirements.txt commands = - find . -type f -name "*.pyc" -delete stestr run --serial --slowest {posargs} -allowlist_externals = find [testenv:integration] setenv = {[testenv]setenv} @@ -31,6 +29,10 @@ setenv = {[testenv]setenv} OS_TEST_PATH=./zaqar/tests/functional commands = stestr run --serial --slowest {posargs} +[testenv:py{3,38,39,310,311}] +install_command = + python -m pip install -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} {opts} {packages} + [testenv:pep8] commands = doc8 doc/source @@ -58,8 +60,10 @@ commands = commands = {posargs} [testenv:docs] +install_command = + python -m pip install -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} {opts} {packages} deps = - -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -r{toxinidir}/requirements.txt -r{toxinidir}/doc/requirements.txt commands = sphinx-build -W -b html doc/source doc/build/html diff --git a/zaqar/storage/mongodb/flavors.py b/zaqar/storage/mongodb/flavors.py index 52984ab9c..f09f7b453 100644 --- a/zaqar/storage/mongodb/flavors.py +++ b/zaqar/storage/mongodb/flavors.py @@ -46,12 +46,17 @@ class FlavorsController(base.FlavorsBase): def __init__(self, *args, **kwargs): super(FlavorsController, self).__init__(*args, **kwargs) - + # To avoid creating unique index twice + flavors_index_str = '_'.join( + map(lambda x: '%s_%s' % (x[0], x[1]), FLAVORS_INDEX) + ) self._col = self.driver.database.flavors - self._col.create_index(FLAVORS_INDEX, - background=True, - name='flavors_name', - unique=True) + indexes = self._col.index_information().keys() + if flavors_index_str and flavors_index_str not in indexes: + self._col.create_index(FLAVORS_INDEX, + background=True, + name='flavors_name', + unique=True) self._col.create_index(FLAVORS_STORAGE_POOL_INDEX, background=True, name='flavors_storage_pool_group_name') diff --git a/zaqar/storage/mongodb/pools.py b/zaqar/storage/mongodb/pools.py index e26e3edb0..f4a2ff355 100644 --- a/zaqar/storage/mongodb/pools.py +++ b/zaqar/storage/mongodb/pools.py @@ -55,17 +55,25 @@ class PoolsController(base.PoolsBase): def __init__(self, *args, **kwargs): super(PoolsController, self).__init__(*args, **kwargs) - + # To avoid creating unique index twice + pools_index_str = '_'.join( + map(lambda x: '%s_%s' % (x[0], x[1]), POOLS_INDEX) + ) + uri_index_str = '_'.join( + map(lambda x: '%s_%s' % (x[0], x[1]), URI_INDEX) + ) self._col = self.driver.database.pools - self._col.create_index(POOLS_INDEX, - background=True, - name='pools_name', - unique=True) - - self._col.create_index(URI_INDEX, - background=True, - name='pools_uri', - unique=True) + indexes = self._col.index_information().keys() + if pools_index_str and pools_index_str not in indexes: + self._col.create_index(POOLS_INDEX, + background=True, + name='pools_name', + unique=True) + if uri_index_str and uri_index_str not in indexes: + self._col.create_index(URI_INDEX, + background=True, + name='pools_uri', + unique=True) @utils.raises_conn_error def _list(self, marker=None, limit=10, detailed=False): diff --git a/zaqar/storage/utils.py b/zaqar/storage/utils.py index 3aac36344..aad475199 100644 --- a/zaqar/storage/utils.py +++ b/zaqar/storage/utils.py @@ -19,7 +19,7 @@ from oslo_config import cfg from oslo_log import log from oslo_serialization import jsonutils from osprofiler import profiler -from stevedore import driver +import stevedore from urllib import parse as urllib_parse from zaqar.common import errors @@ -94,8 +94,9 @@ def load_storage_impl(uri, control_mode=False, default_store=None): storage_type = default_store try: - mgr = driver.DriverManager(driver_type, storage_type, - invoke_on_load=False) + mgr = stevedore.DriverManager(namespace=driver_type, + name=storage_type, + invoke_on_load=False) return mgr.driver @@ -138,10 +139,10 @@ def load_storage_driver(conf, cache, storage_type=None, _invoke_args = (conf, cache, control_driver) try: - mgr = driver.DriverManager(driver_type, - storage_type, - invoke_on_load=True, - invoke_args=_invoke_args) + mgr = stevedore.DriverManager(namespace=driver_type, + name=storage_type, + invoke_on_load=True, + invoke_args=_invoke_args) if conf.profiler.enabled: if ((mode == "control" and conf.profiler.trace_management_store) or