Switch to 2023.1 Python3 unit tests and generic template name

This is an automatically generated patch to ensure unit testing
is in place for all the of the tested runtimes for antelope. Also,
updating the template name to generic one.

See also the PTI in governance [1].

[1]: https://governance.openstack.org/tc/reference/project-testing-interface.html

Change-Id: I29fcb9afa016b1321b0b4c0118a8d7ffbc00b02e

.zuul.yaml

@@ -75,8 +75,8 @@
 
 - project:
     templates:
-      - openstack-python3-zed-jobs
+      - openstack-python3-jobs
-      - openstack-python3-zed-jobs-arm64
+      - openstack-python3-jobs-arm64
       - publish-openstack-docs-pti
       - periodic-stable-jobs
       - check-requirements
@@ -92,16 +92,12 @@
         - zaqar-grenade-mongodb
         - zaqar-grenade-redis
         - zaqar-grenade-swift
-        # Heat job that deploy Zaqar
-        - heat-functional-convg-mysql-lbaasv2
         - zaqar-rally:
             voting: false
     gate:
-      queue: zaqar
       jobs:
         - zaqar-tox-integration
         - zaqar-tempest-plugin-mongodb
         - zaqar-tempest-plugin-redis
         - zaqar-tempest-plugin-swift
         - zaqar-tempest-plugin-swift-ipv6
-        - heat-functional-convg-mysql-lbaasv2

bindep.txt

@@ -3,7 +3,6 @@
 # their own bindep.txt files if the list below is not
 # working for them.
 
-mongodb [platform:dpkg]
 mongodb-server [platform:rpm !platform:opensuseproject]
 dev-db/mongodb [platform:gentoo]
 redis [platform:rpm]

devstack/plugin.sh

@@ -50,15 +50,16 @@ function cleanup_zaqar {
 
 # cleanup_zaqar_mongodb() - Remove residual data files, anything left over from previous
 # runs that a clean run would need to clean up
+# After mongodb 6.0, the mongo shell has been remove, now using mongosh.
 function cleanup_zaqar_mongodb {
-    if ! timeout $SERVICE_TIMEOUT sh -c "while ! mongo zaqar --eval 'db.dropDatabase();'; do sleep 1; done"; then
+    if ! timeout $SERVICE_TIMEOUT sh -c "while ! mongosh zaqar --eval 'db.dropDatabase();'; do sleep 1; done"; then
         die $LINENO "Mongo DB did not start"
     else
-        full_version=$(mongo zaqar --eval 'db.dropDatabase();')
+        full_version=$(mongosh zaqar --eval 'db.dropDatabase();')
-        mongo_version=`echo $full_version | cut -d' ' -f4`
+        mongo_version=`echo $full_version | cut -d' ' -f11`
-        required_mongo_version='2.2'
+        required_mongo_version='6.0'
         if [[ $mongo_version < $required_mongo_version ]]; then
-            die $LINENO "Zaqar needs Mongo DB version >= 2.2 to run."
+            die $LINENO "Zaqar needs Mongo DB version >= 6.0 to run."
         fi
     fi
 }
@@ -186,15 +187,18 @@ function configure_mongodb {
     # per database.
     pip_install pymongo
     if is_ubuntu; then
-        install_package mongodb-server
+        # NOTE: To fix the mongodb's issue in ubuntu 22.04 LTS
-        if ! grep -qF "smallfiles = true" /etc/mongodb.conf; then
+        wget -qO - https://www.mongodb.org/static/pgp/server-6.0.asc | sudo apt-key add -
-            echo "smallfiles = true" | sudo tee --append /etc/mongodb.conf > /dev/null
+        echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/6.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-6.0.list
-        fi
+        sudo apt update
-        restart_service mongodb
+        curl -LO http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1-1ubuntu2.1~18.04.20_amd64.deb
+        sudo dpkg -i ./libssl1.1_1.1.1-1ubuntu2.1~18.04.20_amd64.deb
+
+        install_package mongodb-org
+        restart_service mongod
     elif is_fedora; then
         install_package mongodb
         install_package mongodb-server
-        sudo sed -i '/--smallfiles/!s/OPTIONS=\"/OPTIONS=\"--smallfiles /' /etc/sysconfig/mongod
         restart_service mongod
     fi
 }

requirements.txt

@@ -16,17 +16,17 @@ python-swiftclient>=3.10.1 # Apache-2.0
 WebOb>=1.7.1 # MIT
 stevedore>=3.2.2 # Apache-2.0
 oslo.cache>=1.26.0 # Apache-2.0
-oslo.config>=6.8.0 # Apache-2.0
+oslo.config>=8.3.2 # Apache-2.0
 oslo.context>=2.19.2 # Apache-2.0
-oslo.db>=8.4.0 # Apache-2.0
+oslo.db>=11.0.0 # Apache-2.0
 oslo.i18n>=3.15.3 # Apache-2.0
-oslo.log>=3.36.0 # Apache-2.0
+oslo.log>=4.6.1 # Apache-2.0
-oslo.messaging>=5.29.0 # Apache-2.0
+oslo.messaging>=12.5.0 # Apache-2.0
-oslo.reports>=1.18.0 # Apache-2.0
+oslo.reports>=2.2.0 # Apache-2.0
-oslo.serialization!=2.19.1,>=2.18.0 # Apache-2.0
+oslo.serialization>=4.2.0 # Apache-2.0
 oslo.upgradecheck>=1.3.0 # Apache-2.0
-oslo.utils>=3.33.0 # Apache-2.0
+oslo.utils>=4.12.1 # Apache-2.0
-oslo.policy>=3.6.0 # Apache-2.0
+oslo.policy>=3.8.1 # Apache-2.0
 osprofiler>=1.4.0 # Apache-2.0
 SQLAlchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8,>=1.3.19 # MIT
 autobahn>=22.3.2 # MIT License

tools/test-setup.sh

@@ -31,3 +31,22 @@ mysql -u $DB_USER -p$DB_PW -h 127.0.0.1 -e "
     SET default_storage_engine=MYISAM;
     DROP DATABASE IF EXISTS openstack_citest;
     CREATE DATABASE openstack_citest CHARACTER SET utf8;"
+
+# TO fix the mongodb issue in ubuntu 22.04
+ubuntu_version=`cat /etc/issue | cut -d " "  -f2`
+if [[ $ubuntu_version > '22' ]]; then
+    wget -qO - https://www.mongodb.org/static/pgp/server-6.0.asc | sudo apt-key add -
+    echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/6.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-6.0.list
+    sudo apt update
+    curl -LO http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1-1ubuntu2.1~18.04.20_amd64.deb
+    sudo dpkg -i ./libssl1.1_1.1.1-1ubuntu2.1~18.04.20_amd64.deb
+    sudo apt install -y mongodb-org
+    sudo systemctl restart mongod
+else
+    sudo apt-get install -y mongodb
+    sudo systemctl restart mongodb
+fi
+
+sudo apt install -y pip
+pip install setuptools
+sudo python3 setup.py install

tox.ini

@@ -17,13 +17,11 @@ setenv = VIRTUAL_ENV={envdir}
          OS_STDOUT_CAPTURE=1
          OS_STDERR_CAPTURE=1
          OS_TEST_TIMEOUT=60
-deps = -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
+deps =
-       -r{toxinidir}/requirements.txt
   -r{toxinidir}/test-requirements.txt
+  -r{toxinidir}/requirements.txt
 commands =
-    find . -type f -name "*.pyc" -delete
     stestr run --serial --slowest {posargs}
-allowlist_externals = find
 
 [testenv:integration]
 setenv = {[testenv]setenv}
@@ -31,6 +29,10 @@ setenv = {[testenv]setenv}
          OS_TEST_PATH=./zaqar/tests/functional
 commands = stestr run --serial --slowest {posargs}
 
+[testenv:py{3,38,39,310,311}]
+install_command =
+  python -m pip install -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} {opts} {packages}
+
 [testenv:pep8]
 commands =
   doc8 doc/source
@@ -58,8 +60,10 @@ commands =
 commands = {posargs}
 
 [testenv:docs]
+install_command =
+    python -m pip install -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} {opts} {packages}
 deps =
-    -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
+    -r{toxinidir}/requirements.txt
     -r{toxinidir}/doc/requirements.txt
 commands =
     sphinx-build -W -b html doc/source doc/build/html

zaqar/storage/mongodb/flavors.py

@@ -46,8 +46,13 @@ class FlavorsController(base.FlavorsBase):
 
     def __init__(self, *args, **kwargs):
         super(FlavorsController, self).__init__(*args, **kwargs)
-
+        # To avoid creating unique index twice
+        flavors_index_str = '_'.join(
+            map(lambda x: '%s_%s' % (x[0], x[1]), FLAVORS_INDEX)
+        )
         self._col = self.driver.database.flavors
+        indexes = self._col.index_information().keys()
+        if flavors_index_str and flavors_index_str not in indexes:
             self._col.create_index(FLAVORS_INDEX,
                                    background=True,
                                    name='flavors_name',

zaqar/storage/mongodb/pools.py

@@ -55,13 +55,21 @@ class PoolsController(base.PoolsBase):
 
     def __init__(self, *args, **kwargs):
         super(PoolsController, self).__init__(*args, **kwargs)
-
+        # To avoid creating unique index twice
+        pools_index_str = '_'.join(
+            map(lambda x: '%s_%s' % (x[0], x[1]), POOLS_INDEX)
+        )
+        uri_index_str = '_'.join(
+            map(lambda x: '%s_%s' % (x[0], x[1]), URI_INDEX)
+        )
         self._col = self.driver.database.pools
+        indexes = self._col.index_information().keys()
+        if pools_index_str and pools_index_str not in indexes:
             self._col.create_index(POOLS_INDEX,
                                    background=True,
                                    name='pools_name',
                                    unique=True)
-
+        if uri_index_str and uri_index_str not in indexes:
             self._col.create_index(URI_INDEX,
                                    background=True,
                                    name='pools_uri',

zaqar/storage/utils.py

@@ -19,7 +19,7 @@ from oslo_config import cfg
 from oslo_log import log
 from oslo_serialization import jsonutils
 from osprofiler import profiler
-from stevedore import driver
+import stevedore
 from urllib import parse as urllib_parse
 
 from zaqar.common import errors
@@ -94,7 +94,8 @@ def load_storage_impl(uri, control_mode=False, default_store=None):
         storage_type = default_store
 
     try:
-        mgr = driver.DriverManager(driver_type, storage_type,
+        mgr = stevedore.DriverManager(namespace=driver_type,
+                                      name=storage_type,
                                       invoke_on_load=False)
 
         return mgr.driver
@@ -138,8 +139,8 @@ def load_storage_driver(conf, cache, storage_type=None,
         _invoke_args = (conf, cache, control_driver)
 
     try:
-        mgr = driver.DriverManager(driver_type,
+        mgr = stevedore.DriverManager(namespace=driver_type,
-                                   storage_type,
+                                      name=storage_type,
                                       invoke_on_load=True,
                                       invoke_args=_invoke_args)