diff --git a/.zuul.yaml b/.zuul.yaml index 14ae1bec3..afc8cc95a 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -75,8 +75,8 @@ - project: templates: - - openstack-python3-zed-jobs - - openstack-python3-zed-jobs-arm64 + - openstack-python3-jobs + - openstack-python3-jobs-arm64 - publish-openstack-docs-pti - periodic-stable-jobs - check-requirements @@ -92,16 +92,12 @@ - zaqar-grenade-mongodb - zaqar-grenade-redis - zaqar-grenade-swift - # Heat job that deploy Zaqar - - heat-functional-convg-mysql-lbaasv2 - zaqar-rally: voting: false gate: - queue: zaqar jobs: - zaqar-tox-integration - zaqar-tempest-plugin-mongodb - zaqar-tempest-plugin-redis - zaqar-tempest-plugin-swift - zaqar-tempest-plugin-swift-ipv6 - - heat-functional-convg-mysql-lbaasv2 diff --git a/bindep.txt b/bindep.txt index 5afd238c5..0603c33ed 100644 --- a/bindep.txt +++ b/bindep.txt @@ -3,7 +3,6 @@ # their own bindep.txt files if the list below is not # working for them. -mongodb [platform:dpkg] mongodb-server [platform:rpm !platform:opensuseproject] dev-db/mongodb [platform:gentoo] redis [platform:rpm] diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 788a0ed15..dd81b92b8 100755 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -50,15 +50,16 @@ function cleanup_zaqar { # cleanup_zaqar_mongodb() - Remove residual data files, anything left over from previous # runs that a clean run would need to clean up +# After mongodb 6.0, the mongo shell has been remove, now using mongosh. function cleanup_zaqar_mongodb { - if ! timeout $SERVICE_TIMEOUT sh -c "while ! mongo zaqar --eval 'db.dropDatabase();'; do sleep 1; done"; then + if ! timeout $SERVICE_TIMEOUT sh -c "while ! mongosh zaqar --eval 'db.dropDatabase();'; do sleep 1; done"; then die $LINENO "Mongo DB did not start" else - full_version=$(mongo zaqar --eval 'db.dropDatabase();') - mongo_version=`echo $full_version | cut -d' ' -f4` - required_mongo_version='2.2' + full_version=$(mongosh zaqar --eval 'db.dropDatabase();') + mongo_version=`echo $full_version | cut -d' ' -f11` + required_mongo_version='6.0' if [[ $mongo_version < $required_mongo_version ]]; then - die $LINENO "Zaqar needs Mongo DB version >= 2.2 to run." + die $LINENO "Zaqar needs Mongo DB version >= 6.0 to run." fi fi } @@ -186,15 +187,18 @@ function configure_mongodb { # per database. pip_install pymongo if is_ubuntu; then - install_package mongodb-server - if ! grep -qF "smallfiles = true" /etc/mongodb.conf; then - echo "smallfiles = true" | sudo tee --append /etc/mongodb.conf > /dev/null - fi - restart_service mongodb + # NOTE: To fix the mongodb's issue in ubuntu 22.04 LTS + wget -qO - https://www.mongodb.org/static/pgp/server-6.0.asc | sudo apt-key add - + echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/6.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-6.0.list + sudo apt update + curl -LO http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1-1ubuntu2.1~18.04.20_amd64.deb + sudo dpkg -i ./libssl1.1_1.1.1-1ubuntu2.1~18.04.20_amd64.deb + + install_package mongodb-org + restart_service mongod elif is_fedora; then install_package mongodb install_package mongodb-server - sudo sed -i '/--smallfiles/!s/OPTIONS=\"/OPTIONS=\"--smallfiles /' /etc/sysconfig/mongod restart_service mongod fi } diff --git a/requirements.txt b/requirements.txt index 1d5b0092c..5efcdcacf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,17 +16,17 @@ python-swiftclient>=3.10.1 # Apache-2.0 WebOb>=1.7.1 # MIT stevedore>=3.2.2 # Apache-2.0 oslo.cache>=1.26.0 # Apache-2.0 -oslo.config>=6.8.0 # Apache-2.0 +oslo.config>=8.3.2 # Apache-2.0 oslo.context>=2.19.2 # Apache-2.0 -oslo.db>=8.4.0 # Apache-2.0 +oslo.db>=11.0.0 # Apache-2.0 oslo.i18n>=3.15.3 # Apache-2.0 -oslo.log>=3.36.0 # Apache-2.0 -oslo.messaging>=5.29.0 # Apache-2.0 -oslo.reports>=1.18.0 # Apache-2.0 -oslo.serialization!=2.19.1,>=2.18.0 # Apache-2.0 +oslo.log>=4.6.1 # Apache-2.0 +oslo.messaging>=12.5.0 # Apache-2.0 +oslo.reports>=2.2.0 # Apache-2.0 +oslo.serialization>=4.2.0 # Apache-2.0 oslo.upgradecheck>=1.3.0 # Apache-2.0 -oslo.utils>=3.33.0 # Apache-2.0 -oslo.policy>=3.6.0 # Apache-2.0 +oslo.utils>=4.12.1 # Apache-2.0 +oslo.policy>=3.8.1 # Apache-2.0 osprofiler>=1.4.0 # Apache-2.0 SQLAlchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8,>=1.3.19 # MIT autobahn>=22.3.2 # MIT License diff --git a/tools/test-setup.sh b/tools/test-setup.sh index 3cf57b233..9090c018b 100755 --- a/tools/test-setup.sh +++ b/tools/test-setup.sh @@ -31,3 +31,22 @@ mysql -u $DB_USER -p$DB_PW -h 127.0.0.1 -e " SET default_storage_engine=MYISAM; DROP DATABASE IF EXISTS openstack_citest; CREATE DATABASE openstack_citest CHARACTER SET utf8;" + +# TO fix the mongodb issue in ubuntu 22.04 +ubuntu_version=`cat /etc/issue | cut -d " " -f2` +if [[ $ubuntu_version > '22' ]]; then + wget -qO - https://www.mongodb.org/static/pgp/server-6.0.asc | sudo apt-key add - + echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/6.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-6.0.list + sudo apt update + curl -LO http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1-1ubuntu2.1~18.04.20_amd64.deb + sudo dpkg -i ./libssl1.1_1.1.1-1ubuntu2.1~18.04.20_amd64.deb + sudo apt install -y mongodb-org + sudo systemctl restart mongod +else + sudo apt-get install -y mongodb + sudo systemctl restart mongodb +fi + +sudo apt install -y pip +pip install setuptools +sudo python3 setup.py install diff --git a/tox.ini b/tox.ini index 9eaed719b..e55a69c21 100644 --- a/tox.ini +++ b/tox.ini @@ -17,13 +17,11 @@ setenv = VIRTUAL_ENV={envdir} OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 OS_TEST_TIMEOUT=60 -deps = -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} - -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt +deps = + -r{toxinidir}/test-requirements.txt + -r{toxinidir}/requirements.txt commands = - find . -type f -name "*.pyc" -delete stestr run --serial --slowest {posargs} -allowlist_externals = find [testenv:integration] setenv = {[testenv]setenv} @@ -31,6 +29,10 @@ setenv = {[testenv]setenv} OS_TEST_PATH=./zaqar/tests/functional commands = stestr run --serial --slowest {posargs} +[testenv:py{3,38,39,310,311}] +install_command = + python -m pip install -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} {opts} {packages} + [testenv:pep8] commands = doc8 doc/source @@ -58,8 +60,10 @@ commands = commands = {posargs} [testenv:docs] +install_command = + python -m pip install -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} {opts} {packages} deps = - -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -r{toxinidir}/requirements.txt -r{toxinidir}/doc/requirements.txt commands = sphinx-build -W -b html doc/source doc/build/html diff --git a/zaqar/storage/mongodb/flavors.py b/zaqar/storage/mongodb/flavors.py index 52984ab9c..f09f7b453 100644 --- a/zaqar/storage/mongodb/flavors.py +++ b/zaqar/storage/mongodb/flavors.py @@ -46,12 +46,17 @@ class FlavorsController(base.FlavorsBase): def __init__(self, *args, **kwargs): super(FlavorsController, self).__init__(*args, **kwargs) - + # To avoid creating unique index twice + flavors_index_str = '_'.join( + map(lambda x: '%s_%s' % (x[0], x[1]), FLAVORS_INDEX) + ) self._col = self.driver.database.flavors - self._col.create_index(FLAVORS_INDEX, - background=True, - name='flavors_name', - unique=True) + indexes = self._col.index_information().keys() + if flavors_index_str and flavors_index_str not in indexes: + self._col.create_index(FLAVORS_INDEX, + background=True, + name='flavors_name', + unique=True) self._col.create_index(FLAVORS_STORAGE_POOL_INDEX, background=True, name='flavors_storage_pool_group_name') diff --git a/zaqar/storage/mongodb/pools.py b/zaqar/storage/mongodb/pools.py index e26e3edb0..f4a2ff355 100644 --- a/zaqar/storage/mongodb/pools.py +++ b/zaqar/storage/mongodb/pools.py @@ -55,17 +55,25 @@ class PoolsController(base.PoolsBase): def __init__(self, *args, **kwargs): super(PoolsController, self).__init__(*args, **kwargs) - + # To avoid creating unique index twice + pools_index_str = '_'.join( + map(lambda x: '%s_%s' % (x[0], x[1]), POOLS_INDEX) + ) + uri_index_str = '_'.join( + map(lambda x: '%s_%s' % (x[0], x[1]), URI_INDEX) + ) self._col = self.driver.database.pools - self._col.create_index(POOLS_INDEX, - background=True, - name='pools_name', - unique=True) - - self._col.create_index(URI_INDEX, - background=True, - name='pools_uri', - unique=True) + indexes = self._col.index_information().keys() + if pools_index_str and pools_index_str not in indexes: + self._col.create_index(POOLS_INDEX, + background=True, + name='pools_name', + unique=True) + if uri_index_str and uri_index_str not in indexes: + self._col.create_index(URI_INDEX, + background=True, + name='pools_uri', + unique=True) @utils.raises_conn_error def _list(self, marker=None, limit=10, detailed=False): diff --git a/zaqar/storage/utils.py b/zaqar/storage/utils.py index 3aac36344..aad475199 100644 --- a/zaqar/storage/utils.py +++ b/zaqar/storage/utils.py @@ -19,7 +19,7 @@ from oslo_config import cfg from oslo_log import log from oslo_serialization import jsonutils from osprofiler import profiler -from stevedore import driver +import stevedore from urllib import parse as urllib_parse from zaqar.common import errors @@ -94,8 +94,9 @@ def load_storage_impl(uri, control_mode=False, default_store=None): storage_type = default_store try: - mgr = driver.DriverManager(driver_type, storage_type, - invoke_on_load=False) + mgr = stevedore.DriverManager(namespace=driver_type, + name=storage_type, + invoke_on_load=False) return mgr.driver @@ -138,10 +139,10 @@ def load_storage_driver(conf, cache, storage_type=None, _invoke_args = (conf, cache, control_driver) try: - mgr = driver.DriverManager(driver_type, - storage_type, - invoke_on_load=True, - invoke_args=_invoke_args) + mgr = stevedore.DriverManager(namespace=driver_type, + name=storage_type, + invoke_on_load=True, + invoke_args=_invoke_args) if conf.profiler.enabled: if ((mode == "control" and conf.profiler.trace_management_store) or