Native Zuul v3 version of the grenade jobs

The upgrade pre/post script have been removed, because they are empty.

This change includes a few fixes for the grenade plugin:
- make sure to install mongo-tools when running grenade tests,
  as they use mongodump which is part of that package.
- use sudo also when saving the redis dump file for the target
  logs, as it already happens when the dump for the base logs
  is copied.

The jobs have been ported separately from the tempest and rally
jobs because they can't be backported to all the older stable branches
like them (right now, only up to train).

Change-Id: I73c4c4dee994f13b7411a17f0c5d95ca9c84bd30

.zuul.yaml

@@ -1,39 +1,63 @@
 - job:
     name: zaqar-grenade-base
-    parent: legacy-dsvm-base
+    abstract: true
+    parent: grenade
+    description: |
+      Abstract base grenade job for zaqar tempest plugin
+    required-projects:
+      - opendev.org/openstack/python-zaqarclient
+      - opendev.org/openstack/zaqar
+      - opendev.org/openstack/zaqar-tempest-plugin
+    vars:
+      devstack_plugins:
+        zaqar: https://opendev.org/openstack/zaqar
+      devstack_services:
+        # probably other services can be disabled;
+        # the old legacy job only uses mysql, keystone, zaqar and tempest
+        s-account: false
+        s-container: false
+        s-object: false
+        s-proxy: false
+        c-bak: false
+      tempest_plugins:
+        - zaqar-tempest-plugin
+      tempest_test_regex: zaqar_tempest_plugin.tests
+      tox_envlist: all
     irrelevant-files:
       - ^(test-|)requirements.txt$
       - ^setup.cfg$
-    required-projects:
-      - openstack/grenade
-      - openstack/devstack-gate
-      - openstack/python-zaqarclient
-      - openstack/zaqar
-      - openstack/zaqar-tempest-plugin
-    run: playbooks/legacy/grenade-devstack-zaqar-base/run.yaml
-    post-run: playbooks/legacy/grenade-devstack-zaqar-base/post.yaml
-    timeout: 7800
 
 - job:
     name: zaqar-grenade-mongodb
     parent: zaqar-grenade-base
     voting: false
     vars:
-      database_backend: mongodb
+      grenade_devstack_localrc:
+        shared:
+          ZAQAR_BACKEND: mongodb
 
 - job:
     name: zaqar-grenade-redis
     parent: zaqar-grenade-base
     voting: false
     vars:
-      database_backend: redis
+      grenade_devstack_localrc:
+        shared:
+          ZAQAR_BACKEND: redis
 
 - job:
     name: zaqar-grenade-swift
     parent: zaqar-grenade-base
     voting: false
     vars:
-      database_backend: swift
+      devstack_services:
+        s-account: true
+        s-container: true
+        s-object: true
+        s-proxy: true
+      grenade_devstack_localrc:
+        shared:
+          ZAQAR_BACKEND: swift
 
 - job:
     name: zaqar-tox-integration

devstack/upgrade/upgrade.sh

@@ -35,6 +35,13 @@ source $GRENADE_DIR/functions
 # only the first error that occurred.
 set -o errexit
 
+if grep -q '_store *= *mongodb' /etc/zaqar/zaqar.conf; then
+    # mongo-tools is the name of the package which includes mongodump on
+    # basically all distributions (Ubuntu, Debian, Fedora, CentOS and
+    # openSUSE).
+    install_package mongo-tools
+fi
+
 if grep -q 'management_store *= *mongodb' /etc/zaqar/zaqar.conf; then
     mongodump --db zaqar_mgmt --out $SAVE_DIR/zaqar-mongodb-mgmt-dump.$BASE_RELEASE
 fi
@@ -100,7 +107,7 @@ fi
 
 if grep -q 'message_store *= *redis' /etc/zaqar/zaqar.conf; then
     redis-cli save
-    cp /var/lib/redis/dump.rdb $SAVE_DIR/zaqar-redis-message-dump-$TARGET_RELEASE.rdb
+    sudo cp /var/lib/redis/dump.rdb $SAVE_DIR/zaqar-redis-message-dump-$TARGET_RELEASE.rdb
 fi
 
 set +o xtrace

playbooks/legacy/grenade-devstack-zaqar-base/post.yaml

@@ -1,15 +0,0 @@
-- hosts: primary
-  tasks:
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/logs/**
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs

playbooks/legacy/grenade-devstack-zaqar-base/run.yaml

@@ -1,84 +0,0 @@
-- hosts: all
-  name: Grenade Devstack Zaqar Base
-  tasks:
-
-    - name: Ensure legacy workspace directory
-      file:
-        path: '{{ ansible_user_dir }}/workspace'
-        state: directory
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          cat > clonemap.yaml << EOF
-          clonemap:
-            - name: openstack/devstack-gate
-              dest: devstack-gate
-          EOF
-          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
-              https://opendev.org \
-              openstack/devstack-gate
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          cat << 'EOF' >>"/tmp/dg-local.conf"
-          [[local|localrc]]
-          enable_plugin zaqar https://opendev.org/openstack/zaqar
-          # Enable Zaqar Tempest plugin
-          TEMPEST_PLUGINS='/opt/stack/new/zaqar-tempest-plugin'
-          USE_PYTHON3="True"
-
-          EOF
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          export PROJECTS="openstack/grenade $PROJECTS"
-          export PROJECTS="openstack/python-zaqarclient $PROJECTS"
-          export PYTHONUNBUFFERED=true
-          export ENABLED_SERVICES=tempest
-          export DEVSTACK_GATE_TEMPEST=0
-          export DEVSTACK_GATE_TEMPEST_REGEX="zaqar_tempest_plugin.tests"
-
-          export DEVSTACK_GATE_GRENADE=pullup
-          export GRENADE_PLUGINRC="enable_grenade_plugin zaqar https://opendev.org/openstack/zaqar"
-          export BRANCH_OVERRIDE=default
-          if [ "$BRANCH_OVERRIDE" != "default" ] ; then
-              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
-          fi
-
-          export PROJECTS="openstack/zaqar-tempest-plugin $PROJECTS"
-
-          function pre_test_hook {
-              cd /opt/stack/new/zaqar/zaqar_upgradetests
-              ./pre_test_hook.sh
-          }
-          export -f pre_test_hook
-
-          function post_test_hook {
-              cd /opt/stack/new/zaqar/zaqar_upgradetests
-              ./post_test_hook.sh
-          }
-          export -f post_test_hook
-
-          function gate_hook {
-              cd /opt/stack/new/zaqar/devstack/gate
-              ./gate_hook.sh tempest "{{ database_backend }}"
-          }
-          export -f gate_hook
-
-          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
-          ./safe-devstack-vm-gate-wrap.sh
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'