[CI] Improve CI tests for mysql

As the previous jobs always timeout in the CI system, This commit separates the task to multiple tests to prevent timeout. This commit also adds replication tests for mysql Depends-On: https://review.opendev.org/c/openstack/trove-tempest-plugin/+/899777 Change-Id: Ie555be434f773ac9e48022dd2ff763a16c2aaee7
2023-10-26 10:20:55 +08:00 · 2023-10-26 10:20:55 +08:00 · aaaad924b0
commit aaaad924b0
parent b5e85ae2ae
4 changed files with 101 additions and 26 deletions
--- a/devstack/plugin.sh
+++ b/devstack/plugin.sh
@ -532,13 +532,13 @@ function create_registry_container {
        done
        pushd $DEST/trove/backup
        # build backup images
-        sudo docker build --network host -t 127.0.0.1:4000/trove-datastores/db-backup-mysql5.7:1.1.0 --build-arg DATASTORE=mysql --build-arg DATASTORE_VERSION=5.7 .
-        sudo docker build --network host -t 127.0.0.1:4000/trove-datastores/db-backup-mysql8.0:1.1.0 --build-arg DATASTORE=mysql --build-arg DATASTORE_VERSION=8.0 .
-        sudo docker build --network host -t 127.0.0.1:4000/trove-datastores/db-backup-mariadb:1.1.0 --build-arg DATASTORE=mariadb --build-arg DATASTORE_VERSION=10.4 .
-        sudo docker build --network host -t 127.0.0.1:4000/trove-datastores/db-backup-postgresql:1.1.2 --build-arg DATASTORE=postgresql --build-arg DATASTORE_VERSION=12 .
+        sudo docker build --network host -t 127.0.0.1:4000/trove-datastores/db-backup-mysql:5.7 --build-arg DATASTORE=mysql --build-arg DATASTORE_VERSION=5.7 .
+        sudo docker build --network host -t 127.0.0.1:4000/trove-datastores/db-backup-mysql:8.0 --build-arg DATASTORE=mysql --build-arg DATASTORE_VERSION=8.0 .
+        sudo docker build --network host -t 127.0.0.1:4000/trove-datastores/db-backup-mariadb:10.4 --build-arg DATASTORE=mariadb --build-arg DATASTORE_VERSION=10.4 .
+        sudo docker build --network host -t 127.0.0.1:4000/trove-datastores/db-backup-postgresql:12 --build-arg DATASTORE=postgresql --build-arg DATASTORE_VERSION=12 .
        popd
        # push backup images
-        for backupimg in {"db-backup-mysql5.7:1.1.0","db-backup-mysql8.0:1.1.0","db-backup-mariadb:1.1.0","db-backup-postgresql:1.1.2"};
+        for backupimg in {"db-backup-mysql:5.7","db-backup-mysql:8.0","db-backup-mariadb:10.4","db-backup-postgresql:12"};
        do
        sudo docker push 127.0.0.1:4000/trove-datastores/${backupimg}
        done
--- a/devstack/settings
+++ b/devstack/settings
@ -33,7 +33,7 @@ TROVE_LOCAL_POLICY_JSON=${TROVE_LOCAL_POLICY_JSON:-${TROVE_LOCAL_CONF_DIR}/polic
 TROVE_IMAGE_OS=${TROVE_IMAGE_OS:-"ubuntu"}
 TROVE_IMAGE_OS_RELEASE=${TROVE_IMAGE_OS_RELEASE:-"jammy"}
 TROVE_DATASTORE_TYPE=${TROVE_DATASTORE_TYPE:-"mysql"}
-TROVE_DATASTORE_VERSION=${TROVE_DATASTORE_VERSION:-"5.7.29"}
+TROVE_DATASTORE_VERSION=${TROVE_DATASTORE_VERSION:-"5.7"}

 # Configuration values listed here for reference
 TROVE_MAX_ACCEPTED_VOLUME_SIZE=${TROVE_MAX_ACCEPTED_VOLUME_SIZE}
@ -94,12 +94,12 @@ if [ "$TROVE_ENABLE_LOCAL_REGISTRY" == "True" ] ; then
    TROVE_DATABASE_IMAGE_MYSQL=${TROVE_DATABASE_IMAGE_MYSQL:-"$TROVE_HOST_GATEWAY:4000/trove-datastores/mysql"}
    TROVE_DATABASE_IMAGE_MARIADB=${TROVE_DATABASE_IMAGE_MARIAD:-"$TROVE_HOST_GATEWAY:4000/trove-datastores/mariadb"}
    TROVE_DATABASE_IMAGE_POSTGRES=${TROVE_DATABASE_IMAGE_POSTGRES:-"$TROVE_HOST_GATEWAY:4000/trove-datastores/postgres"}
-    TROVE_DATABASE_BACKUP_IMAGE_MYSQL=${TROVE_DATABASE_BACKUP_IMAGE_MYSQL:-"$TROVE_HOST_GATEWAY:4000/trove-datastores/db-backup-mysql:1.1.0"}
-    TROVE_DATABASE_BACKUP_IMAGE_MARIADB=${TROVE_DATABASE_BACKUP_IMAGE_MARIADB:-"$TROVE_HOST_GATEWAY:4000/trove-datastores/db-backup-mariadb:1.1.0"}
-    TROVE_DATABASE_BACKUP_IMAGE_POSTGRES=${TROVE_DATABASE_BACKUP_IMAGE_POSTGRES:-"$TROVE_HOST_GATEWAY:4000/trove-datastores/db-backup-postgresql:1.1.2"}
+    TROVE_DATABASE_BACKUP_IMAGE_MYSQL=${TROVE_DATABASE_BACKUP_IMAGE_MYSQL:-"$TROVE_HOST_GATEWAY:4000/trove-datastores/db-backup-mysql"}
+    TROVE_DATABASE_BACKUP_IMAGE_MARIADB=${TROVE_DATABASE_BACKUP_IMAGE_MARIADB:-"$TROVE_HOST_GATEWAY:4000/trove-datastores/db-backup-mariadb"}
+    TROVE_DATABASE_BACKUP_IMAGE_POSTGRES=${TROVE_DATABASE_BACKUP_IMAGE_POSTGRES:-"$TROVE_HOST_GATEWAY:4000/trove-datastores/db-backup-postgresql"}
 else
-    TROVE_DATABASE_IMAGE_MYSQL=${TROVE_DATABASE_IMAGE_MYSQL:-"catalystcloud/mysql"}
+    TROVE_DATABASE_IMAGE_MYSQL=${TROVE_DATABASE_IMAGE_MYSQL:-"docker.io/library/mysql"}
    TROVE_DATABASE_IMAGE_POSTGRES=${TROVE_DATABASE_IMAGE_POSTGRES:-"catalystcloud/postgres"}
-    TROVE_DATABASE_BACKUP_IMAGE_MYSQL=${TROVE_DATABASE_BACKUP_IMAGE_MYSQL:-"catalystcloud/db-backup-mysql:1.1.0"}
+    TROVE_DATABASE_BACKUP_IMAGE_MYSQL=${TROVE_DATABASE_BACKUP_IMAGE_MYSQL:-""}
    TROVE_DATABASE_BACKUP_IMAGE_POSTGRES=${TROVE_DATABASE_BACKUP_IMAGE_POSTGRES:-"catalystcloud/db-backup-postgresql:1.1.2"}
 fi
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@ -298,19 +298,22 @@
        Q_PLUGIN: ml2
        Q_ML2_TENANT_NETWORK_TYPE: vxlan
        Q_ML2_PLUGIN_MECHANISM_DRIVERS: openvswitch
-        TROVE_ENABLE_LOCAL_REGISTRY: True
        SYNC_LOG_TO_CONTROLLER: True
        TROVE_DATASTORE_VERSION: 5.7
+        TROVE_MGMT_GATEWAY: "192.168.254.1"
      devstack_local_conf:
        post-config:
          $TROVE_CONF:
            DEFAULT:
              usage_timeout: 1800
+            network:
+              network_isolation: True
        test-config:
          $TEMPEST_CONFIG:
            auth:
              tempest_roles: ResellerAdmin
            database:
+              database_build_timeout: 2400
              default_datastore_versions: mysql:5.7
      devstack_plugins:
        trove: https://opendev.org/openstack/trove.git
@ -345,24 +348,99 @@
        ovn-northd: false
      tempest_test_regex: ^trove_tempest_plugin\.tests
      tempest_test_timeout: 3600
-      tempest_test_exclude_list: '{{ ansible_user_dir }}/{{ zuul.projects["opendev.org/openstack/trove-tempest-plugin"].src_dir }}/tempest_exclude_list.txt'
      zuul_copy_output:
        '/var/log/guest-agent-logs/': 'logs'

 - job:
-    name: trove-tempest-mysql-network-isolation
+    name: trove-tempest-ubuntu-base
+    parent: trove-tempest
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^api-ref/.*$
+      - ^doc/.*$
+      - ^etc/.*$
+      - ^releasenotes/.*$
+      - ^test-requirements.txt$
+      - ^tox.ini$
+      - ^LICENSE$
+      - ^contrib/
+      - ^zuul\.d/
+      - ^backup/
+      - ^\..+
+    vars:
+      devstack_services:
+        swift: false
+        s-account: false
+        s-container: false
+        s-object: false
+        s-proxy: false
+      tempest_test_regex: ^trove_tempest_plugin\.tests\.scenario\.test_instance
+
+- job:
+    name: trove-tempest-ubuntu-backup
    parent: trove-tempest
    vars:
      devstack_localrc:
        TROVE_ENABLE_LOCAL_REGISTRY: True
-        TROVE_MGMT_GATEWAY: "192.168.254.1"
-      devstack_local_conf:
-        post-config:
-          $TROVE_CONF:
-            network:
-              network_isolation: True
      tempest_test_regex: ^trove_tempest_plugin\.tests\.scenario\.test_backup

+- job:
+    name: trove-tempest-ubuntu-replication
+    parent: trove-tempest
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^api-ref/.*$
+      - ^doc/.*$
+      - ^etc/.*$
+      - ^releasenotes/.*$
+      - ^test-requirements.txt$
+      - ^tox.ini$
+      - ^LICENSE$
+      - ^contrib/
+      - ^zuul\.d/
+      - ^backup/
+      - ^\..+
+    vars:
+      devstack_localrc:
+        TROVE_ENABLE_LOCAL_REGISTRY: True
+      tempest_test_regex: ^trove_tempest_plugin\.tests\.scenario\.test_replication
+
+- job:
+    name: trove-tempest-ubuntu-base-mysql5.7
+    parent: trove-tempest-ubuntu-base
+    vars:
+        devstack_localrc:
+          TROVE_DATASTORE_VERSION: 5.7
+        devstack_local_conf:
+          test-config:
+            $TEMPEST_CONFIG:
+              database:
+                default_datastore_versions: mysql:5.7
+
+- job:
+    name: trove-tempest-ubuntu-backup-mysql5.7
+    parent: trove-tempest-ubuntu-backup
+    vars:
+        devstack_localrc:
+          TROVE_DATASTORE_VERSION: 5.7
+        devstack_local_conf:
+          test-config:
+            $TEMPEST_CONFIG:
+              database:
+                default_datastore_versions: mysql:5.7
+
+- job:
+    name: trove-tempest-ubuntu-replication-mysql5.7
+    parent: trove-tempest-ubuntu-replication
+    vars:
+        devstack_localrc:
+          TROVE_DATASTORE_VERSION: 5.7
+        devstack_local_conf:
+          test-config:
+            $TEMPEST_CONFIG:
+              database:
+                default_datastore_versions: mysql:5.7
+
 - job:
    name: trove-tempest-postgres
    parent: devstack-tempest
--- a/zuul.d/projects.yaml
+++ b/zuul.d/projects.yaml
@ -11,20 +11,17 @@
      - release-notes-jobs-python3
    check:
      jobs:
+        - trove-tempest-ubuntu-base-mysql5.7
+        - trove-tempest-ubuntu-backup-mysql5.7
+        - trove-tempest-ubuntu-replication-mysql5.7
        - openstack-tox-cover:
            voting: true
        - trove-tox-bandit-baseline:
            voting: true
-        - trove-tempest:
-            voting: false
-        - trove-tempest-mysql-network-isolation:
-            voting: true
        - trove-ubuntu-guest-image-build:
            voting: true
        - trove-centos8s-guest-image-build:
            voting: true
-        - trove-tempest-postgres:
-            voting: false
    gate:
      jobs:
        - trove-tempest: