Fix consistency job and guideline inconsistencies

Adds a job which runs all next tests on a devstack master and
experimental jobs which run the same on supported stable releases.
Fix names of a few manila_tempest_tests and designate_tempest_plugin
tests so that they match the names from the plugins.

Change-Id: Iedbaa09e425a29156304eea63b66a1ce69dd77bd

add-ons/dns.2020.11.json

@@ -155,7 +155,7 @@
         "designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_show_zone": {
           "idempotent_id": "id-02ca5d6a-86ce-4f02-9d94-9e5db55c3055"
         },
-        "designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_list_zone": {
+        "designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_list_zones": {
           "idempotent_id": "id-5bfa3cfe-5bc8-443b-bf48-cfba44cbb247"
         },
         "designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_update_zone": {

add-ons/dns.next.json

@@ -155,7 +155,7 @@
         "designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_show_zone": {
           "idempotent_id": "id-02ca5d6a-86ce-4f02-9d94-9e5db55c3055"
         },
-        "designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_list_zone": {
+        "designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_list_zones": {
           "idempotent_id": "id-5bfa3cfe-5bc8-443b-bf48-cfba44cbb247"
         },
         "designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_update_zone": {

add-ons/orchestration.2020.11.json

@@ -637,7 +637,7 @@
         "heat_tempest_plugin.tests.functional.test_preview.StackPreviewTest.test_nested_pass": {
           "idempotent_id": "id-0449113c-ff90-4f2b-8825-27ea35c1983f"
         },
-        "heat_tempest_plugin.tests.functional.test_preview.StackPreviewTest.test_res_group_with_nested_template:Wherej": {
+        "heat_tempest_plugin.tests.functional.test_preview.StackPreviewTest.test_res_group_with_nested_template": {
           "idempotent_id": "id-6ca8ddfc-106f-4ecc-83f7-fca31d0c85ca"
         }
       }

add-ons/orchestration.next.json

@@ -637,7 +637,7 @@
         "heat_tempest_plugin.tests.functional.test_preview.StackPreviewTest.test_nested_pass": {
           "idempotent_id": "id-0449113c-ff90-4f2b-8825-27ea35c1983f"
         },
-        "heat_tempest_plugin.tests.functional.test_preview.StackPreviewTest.test_res_group_with_nested_template:Wherej": {
+        "heat_tempest_plugin.tests.functional.test_preview.StackPreviewTest.test_res_group_with_nested_template": {
           "idempotent_id": "id-6ca8ddfc-106f-4ecc-83f7-fca31d0c85ca"
         }
       }

add-ons/shared_file_system.2020.11.json

@@ -347,7 +347,7 @@
       "project": "manila",
       "required_since": "2020.11",
       "tests": {
-        "manila_tempest_tests.tests.api.test_share_networks.ShareNetworksTest.test_list_share_networks": {
+        "manila_tempest_tests.tests.api.test_share_networks.ShareNetworkListMixin.test_list_share_networks": {
           "idempotent_id": "id-41c635b1-d9ef-4c05-9100-5e4b0034b523"
         }
       }

add-ons/shared_file_system.next.json

@@ -344,7 +344,7 @@
       "project": "manila",
       "required_since": "2020.11",
       "tests": {
-        "manila_tempest_tests.tests.api.test_share_networks.ShareNetworksTest.test_list_share_networks": {
+        "manila_tempest_tests.tests.api.test_share_networks.ShareNetworkListMixin.test_list_share_networks": {
           "idempotent_id": "id-41c635b1-d9ef-4c05-9100-5e4b0034b523"
         }
       }

playbooks/parse_next_tests.yaml

@@ -0,0 +1,13 @@
+---
+- hosts: all
+  tasks:
+    - name: Ensure tox is installed
+      include_role:
+        name: ensure-tox
+      vars:
+        ensure_global_symlinks: true
+
+    - name: Get all next tests
+      command: tox -vv -enext -- --interop-repo {{ ansible_user_dir }}/{{ zuul.projects['opendev.org/osf/interop'].src_dir }}
+      args:
+        chdir: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/osf/interop'].src_dir }}"

test-requirements.txt

@@ -4,3 +4,4 @@ doc8
 jsonschema
 Sphinx
 sphinxcontrib.datatemplates
+refstack>=2.0.0

tools/checktests.py

@@ -112,10 +112,11 @@ def get_tests(module_name):
                     if (classnode.__class__ is ast.FunctionDef and
                             classnode.name.startswith('test_')):
                         for decorator in classnode.decorator_list:
-                            if decorator.func.attr == 'idempotent_id':
+                            if hasattr(decorator, 'func'):
-                                tests['id-' + decorator.args[0].s] = \
+                                if decorator.func.attr == 'idempotent_id':
-                                    module_name + "." + node.name + "." + \
+                                    tests['id-' + decorator.args[0].s] = \
-                                    classnode.name
+                                        module_name + "." + node.name + "." + \
+                                        classnode.name
     return tests
 
 

tools/consistency.sh

@@ -1,40 +1,114 @@
 #!/bin/bash
 
 # This script will run consistency checks for Tempest tests against
-# the three latest interoperability guidelines. It can run in two
+# the current and next interoperability guidelines. It can run in two
 # modes.
 #
-# * If no arguments are specified, the script will check out Tempest
+# * If no arguments are specified, the script will check out Tempest and
-#   into a temporary directory, run the consistency checks, then delete
+#   tempest plugins into a temporary directory, run the consistency checks,
-#   temporary checkout.
+#   then delete temporary checkout.
 #
 # * If an argument is given, this script will assume that it is a
 #   user checked-out repository and run the consistency checks against
 #   that, and leave the directory unchanged on exit. This mode is useful
-#   for gate jobs and Tempest development.
+#   for gate jobs and Tempest/tempest plugin development.
 
 set -x
 
-if [ ! $@ ]; then
+# Prints help
-  TEMPESTDIR=$(mktemp -d)
+function usage {
-  git clone https://opendev.org/openstack/tempest $TEMPESTDIR
+    SCRIPT_NAME="basename ${BASH_SOURCE[0]}"
-  CLEANTEMPEST=cleantempest
+    echo "Usage: ${SCRIPT_NAME} [OPTION]..."
-else
+    echo "Consistency check"
-  TEMPESTDIR=${1}
+    echo ""
+    echo "  -h  Print this usage message"
+    echo "  -t  Local Tempest directory"
+    echo "  -d  Local designate-tempest-plugin directory"
+    echo "  -o  Local heat-tempest-plugin directory"
+    echo "  -s  Local manila-tempest-plugin directory"
+    echo "  -c  Set if tempest and plugins directory should be removed after"
+    echo "      the consistency check"
+    exit 1
+}
+
+while getopts t:d:o:s:ch FLAG; do
+    case ${FLAG} in
+        t)
+            TEMPESTDIR=${OPTARG}
+            ;;
+        d)
+            DNSDIR=${OPTARG}
+            ;;
+        o)
+            ORCHESTRATIONDIR=${OPTARG}
+            ;;
+        s)
+            SFSDIR=${OPTARG}
+            ;;
+        c)
+            CLEANTEMPEST=true
+            ;;
+        h)  #show help
+            usage
+            ;;
+        \?) #unrecognized option - show help
+            echo -e \\n"Option -$OPTARG not allowed."
+            usage
+            ;;
+    esac
+done
+
+# check if a local directory was given (for Tempest or one of the plugins),
+# if not, create a temp dir and clone the project there
+if [[ -z $TEMPESTDIR ]]; then
+    TEMPESTDIR=$(mktemp -d)
+    git clone https://opendev.org/openstack/tempest $TEMPESTDIR
+fi
+if [[ -z $DNSDIR ]]; then
+    DNSDIR=$(mktemp -d)
+    git clone https://opendev.org/openstack/designate-tempest-plugin $DNSDIR
+fi
+if [[ -z $ORCHESTRATIONDIR ]]; then
+    ORCHESTRATIONDIR=$(mktemp -d)
+    git clone https://opendev.org/openstack/heat-tempest-plugin $ORCHESTRATIONDIR
+fi
+if [[ -z $SFSDIR ]]; then
+    SFSDIR=$(mktemp -d)
+    git clone https://opendev.org/openstack/manila-tempest-plugin $SFSDIR
 fi
 
-PYTHONPATH=$TEMPESTDIR python ./tools/checktests.py --guideline next.json
+
+export PYTHONPATH=$TEMPESTDIR:$DNSDIR:$ORCHESTRATIONDIR:$SFSDIR
+
+python3 ./tools/checktests.py --guideline next.json
 exit_1=$?
-
+python3 ./tools/checktests.py --guideline add-ons/dns.next.json --testlib designate_tempest_plugin
-PYTHONPATH=$TEMPESTDIR python ./tools/checktests.py --guideline 2018.02.json
 exit_2=$?
-
+python3 ./tools/checktests.py --guideline add-ons/shared_file_system.next.json --testlib manila_tempest_tests
-PYTHONPATH=$TEMPESTDIR python ./tools/checktests.py --guideline 2018.11.json
 exit_3=$?
+# TODO(kopecmartin) consistency check of heat_tempest_plugin is omitted intentionally until we improve the
+# checktests.py so that it detects ids of the heat_tempest_plugin.api tests which don't use decorator.idempotent_id
+# call to track the id
+# python3 ./tools/checktests.py --guideline add-ons/orchestration.next.json --testlib heat_tempest_plugin
+# exit_4=$?
 
-if [[ ! -z "${CLEANTEMPEST}" ]]; then
+python3 ./tools/checktests.py --guideline 2020.11.json
+exit_5=$?
+python3 ./tools/checktests.py --guideline add-ons/dns.2020.11.json --testlib designate_tempest_plugin
+exit_6=$?
+python3 ./tools/checktests.py --guideline add-ons/shared_file_system.2020.11.json --testlib manila_tempest_tests
+exit_7=$?
+# python3 ./tools/checktests.py --guideline add-ons/orchestration.2020.11.json --testlib heat_tempest_plugin
+# exit_8=$?
+
+
+if [[ "${CLEANTEMPEST}" ]]; then
   rm -rf $TEMPESTDIR
+  rm -rf $DNSDIR
+  rm -rf $ORCHESTRATIONDIR
+  rm -rf $SFSDIR
 fi
 
 
-! (( $exit_1 || $exit_2 || $exit_3 ))
+#! (( $exit_1 || $exit_2 || $exit_3 || $exit_4 || $exit_5 || $exit_6 || $exit_7 || $exit_8 ))
+! (( $exit_1 || $exit_2 || $exit_3 || $exit_5 || $exit_6 || $exit_7 ))

tools/parse_next_tests.py

@@ -0,0 +1,69 @@
+# Copyright (c) 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import json
+import os
+
+from refstack.api.guidelines import Guidelines
+
+BASE_URL = "https://opendev.org/api/v1/repos/osf/interop/"
+REPO_URL = BASE_URL + "contents/previous_guidelines"
+RAW_URL = "https://opendev.org/api/v1/repos/osf/interop/raw/"
+ADDITIONAL_CAPABILITY_URLS = BASE_URL + "contents/add-ons/previous_guidelines"
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(__doc__)
+    parser.add_argument('--out', default="./all_next_tests.txt",
+                        help="Name of the file where all next tests will be"
+                             "written to.")
+    parser.add_argument('--interop-repo', default="./",
+                        help="A path to the local interop repository the "
+                             "script will look for the next files in.")
+    return parser.parse_args()
+
+
+def parse_tests(f_path, target):
+    g = Guidelines(repo_url=REPO_URL, raw_url=RAW_URL,
+                   additional_capability_urls=ADDITIONAL_CAPABILITY_URLS)
+    f = open(f_path, 'r')
+    load_f = json.load(f)
+    caps = g.get_target_capabilities(load_f, target=target)
+    tests = g.get_test_list(load_f, caps)
+    return tests
+
+
+def main():
+    args = parse_arguments()
+    tests = parse_tests(
+        os.path.join(args.interop_repo, 'next.json'), 'platform')
+    tests += parse_tests(
+        os.path.join(args.interop_repo, 'add-ons/dns.next.json'), 'dns')
+    tests += parse_tests(
+        os.path.join(args.interop_repo, 'add-ons/orchestration.next.json'),
+        'orchestration')
+    tests += parse_tests(
+        os.path.join(args.interop_repo,
+                     'add-ons/shared_file_system.next.json'),
+        'shared_file_system')
+    f = open(args.out, 'w')
+    for t in tests:
+        f.write(t + "\n")
+    f.close()
+
+
+if __name__ == "__main__":
+    main()

tox.ini

@@ -109,3 +109,6 @@ commands=
 whitelist_externals =
   sh
   bash
+
+[testenv:next]
+commands=python3 tools/parse_next_tests.py {posargs}

zuul.d/jobs.yaml

@@ -5,8 +5,63 @@
       Ensure consistency between Tempest source and current interop guidelines.
     vars:
       tox_envlist: consistency
-      tox_extra_args: "-vv -- {{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/tempest'].src_dir }}"
+      tox_extra_args: >
+        -vv --
+        -t {{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/tempest'].src_dir }}
+        -d {{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/designate-tempest-plugin'].src_dir }}
+        -o {{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/heat-tempest-plugin'].src_dir }}
+        -s {{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/manila-tempest-plugin'].src_dir }}
       zuul_work_dir: src/opendev.org/osf/interop
     required-projects:
       - name: openstack/tempest
+      - name: openstack/designate-tempest-plugin
+      - name: openstack/heat-tempest-plugin
+      - name: openstack/manila-tempest-plugin
       - name: osf/interop
+
+- job:
+    name: interop-next-refstack-client-master
+    parent: refstack-client-devstack-master
+    description: |
+      A job running refstack-client on a devstack environment with the next guideline.
+    required-projects:
+      - name: osf/interop
+    pre-run: playbooks/parse_next_tests.yaml
+    vars:
+      test_list: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/osf/interop'].src_dir }}/all_next_tests.txt"
+
+- job:
+    name: interop-next-refstack-client-wallaby
+    parent: interop-next-refstack-client-master
+    description: |
+      A job running refstack-client on a devstack wallaby environment with the next guideline.
+    override-checkout: stable/wallaby
+    vars:
+      branch: stable/wallaby
+
+- job:
+    name: interop-next-refstack-client-victoria
+    parent: interop-next-refstack-client-master
+    description: |
+      A job running refstack-client on a devstack victoria environment with the next guideline.
+    override-checkout: stable/victoria
+    vars:
+      branch: stable/victoria
+
+- job:
+    name: interop-next-refstack-client-ussuri
+    parent: interop-next-refstack-client-master
+    description: |
+      A job running refstack-client on a devstack ussuri environment with the next guideline.
+    override-checkout: stable/ussuri
+    vars:
+      branch: stable/ussuri
+
+- job:
+    name: interop-next-refstack-client-train
+    parent: interop-next-refstack-client-master
+    description: |
+      A job running refstack-client on a devstack train environment with the next guideline.
+    override-checkout: stable/train
+    vars:
+      branch: stable/train

zuul.d/projects.yaml

@@ -5,6 +5,14 @@
       jobs:
         - openstack-tox-pep8
         - interop-tempest-consistency
+        # TODO(kopecmartin) master job has issues with ensure-tox role atm
+        # which we will figure out in a following patch. Let's comment out
+        # the job now so that it doesn't block us
+        # - interop-next-refstack-client-master
+        - interop-next-refstack-client-wallaby
+        - interop-next-refstack-client-victoria
+        - interop-next-refstack-client-ussuri
+        - interop-next-refstack-client-train
     gate:
       jobs:
         - openstack-tox-pep8