Revert "Reorganize guidelines and improve consistency.sh"

This reverts commit a9fbc452a6.

Reason for revert: refstack server gives 404 on the guidelines: https://refstack.openstack.org/#/guidelines .. seems like https://review.opendev.org/c/osf/refstack/+/790940 didn't handle the update of the guidelines location everywhere - I suspect that some changes in refstack-ui are needed as well, ah

Change-Id: I2685eb8fccb4dbdcf42683f168ee2cb316137196

add-ons/dns_current_guideline

@@ -1 +0,0 @@
-previous_guidelines/dns.2020.11.json

add-ons/orchestration_current_guideline

@@ -1 +0,0 @@
-previous_guidelines/orchestration.2020.11.json

add-ons/shared_file_system_current_guideline

@@ -1 +0,0 @@
-previous_guidelines/shared_file_system.2020.11.json

current_guideline

@@ -1 +0,0 @@
-previous_guidelines/2020.11.json

tools/consistency.sh

@@ -1,113 +1,40 @@
 #!/bin/bash
 
 # This script will run consistency checks for Tempest tests against
-# the current and next interoperability guidelines. It can run in two
+# the three latest interoperability guidelines. It can run in two
 # modes.
 #
-# * If no arguments are specified, the script will check out Tempest and
+# * If no arguments are specified, the script will check out Tempest
-#   tempest plugins into a temporary directory, run the consistency checks,
+#   into a temporary directory, run the consistency checks, then delete
-#   then delete temporary checkout.
+#   temporary checkout.
 #
 # * If an argument is given, this script will assume that it is a
 #   user checked-out repository and run the consistency checks against
 #   that, and leave the directory unchanged on exit. This mode is useful
-#   for gate jobs and Tempest/tempest plugin development.
+#   for gate jobs and Tempest development.
 
 set -x
 
-# Prints help
+if [ ! $@ ]; then
-function usage {
+  TEMPESTDIR=$(mktemp -d)
-    SCRIPT_NAME="basename ${BASH_SOURCE[0]}"
+  git clone https://opendev.org/openstack/tempest $TEMPESTDIR
-    echo "Usage: ${SCRIPT_NAME} [OPTION]..."
+  CLEANTEMPEST=cleantempest
-    echo "Consistency check"
+else
-    echo ""
+  TEMPESTDIR=${1}
-    echo "  -h  Print this usage message"
-    echo "  -t  Local Tempest directory"
-    echo "  -d  Local designate-tempest-plugin directory"
-    echo "  -o  Local heat-tempest-plugin directory"
-    echo "  -s  Local manila-tempest-plugin directory"
-    echo "  -c  Set if tempest and plugins directory should be removed after"
-    echo "      the consistency check"
-    exit 1
-}
-
-while getopts t:d:o:s:ch FLAG; do
-    case ${FLAG} in
-        t)
-            TEMPESTDIR=${OPTARG}
-            ;;
-        d)
-            DNSDIR=${OPTARG}
-            ;;
-        o)
-            ORCHESTRATIONDIR=${OPTARG}
-            ;;
-        s)
-            SFSDIR=${OPTARG}
-            ;;
-        c)
-            CLEANTEMPEST=true
-            ;;
-        h)  #show help
-            usage
-            ;;
-        \?) #unrecognized option - show help
-            echo -e \\n"Option -$OPTARG not allowed."
-            usage
-            ;;
-    esac
-done
-
-# check if a local directory was given (for Tempest or one of the plugins),
-# if not, create a temp dir and clone the project there
-if [[ -z $TEMPESTDIR ]]; then
-    TEMPESTDIR=$(mktemp -d)
-    git clone https://opendev.org/openstack/tempest $TEMPESTDIR
-fi
-if [[ -z $DNSDIR ]]; then
-    DNSDIR=$(mktemp -d)
-    git clone https://opendev.org/openstack/designate-tempest-plugin $DNSDIR
-fi
-if [[ -z $ORCHESTRATIONDIR ]]; then
-    ORCHESTRATIONDIR=$(mktemp -d)
-    git clone https://opendev.org/openstack/heat-tempest-plugin $ORCHESTRATIONDIR
-fi
-if [[ -z $SFSDIR ]]; then
-    SFSDIR=$(mktemp -d)
-    git clone https://opendev.org/openstack/manila-tempest-plugin $SFSDIR
 fi
 
-
+PYTHONPATH=$TEMPESTDIR python ./tools/checktests.py --guideline next.json
-export PYTHONPATH=$TEMPESTDIR:$DNSDIR:$ORCHESTRATIONDIR:$SFSDIR
-
-python3 ./tools/checktests.py --guideline next.json
 exit_1=$?
-# TODO(kopecmartin) consistency check is commented out temporarily while we fix
-# inconsistency issues in the follow-up patches
-# python3 ./tools/checktests.py --guideline add-ons/dns.next.json --testlib designate_tempest_plugin
-# exit_2=$?
-# python3 ./tools/checktests.py --guideline add-ons/orchestration.next.json --testlib heat_tempest_plugin
-# exit_3=$?
-# python3 ./tools/checktests.py --guideline add-ons/shared_file_system.next.json --testlib manila_tempest_tests
-# exit_4=$?
 
-python3 ./tools/checktests.py --guideline current_guideline
+PYTHONPATH=$TEMPESTDIR python ./tools/checktests.py --guideline 2018.02.json
-exit_5=$?
+exit_2=$?
-# python3 ./tools/checktests.py --guideline add-ons/dns_current_guideline --testlib designate_tempest_plugin
-# exit_6=$?
-# python3 ./tools/checktests.py --guideline add-ons/orchestration_current_guideline --testlib heat_tempest_plugin
-# exit_7=$?
-# python3 ./tools/checktests.py --guideline add-ons/shared_file_system_current_guideline --testlib manila_tempest_tests
-# exit_8=$?
 
+PYTHONPATH=$TEMPESTDIR python ./tools/checktests.py --guideline 2018.11.json
+exit_3=$?
 
-if [[ "${CLEANTEMPEST}" ]]; then
+if [[ ! -z "${CLEANTEMPEST}" ]]; then
   rm -rf $TEMPESTDIR
-  rm -rf $DNSDIR
-  rm -rf $ORCHESTRATIONDIR
-  rm -rf $SFSDIR
 fi
 
 
-# ! (( $exit_1 || $exit_2 || $exit_3 || $exit_4 || $exit_5 || $exit_6 || $exit_7 || $exit_8 ))
+! (( $exit_1 || $exit_2 || $exit_3 ))
-! (( $exit_1 || $exit_5 ))

tox.ini

@@ -45,73 +45,63 @@ exclude = .venv,.git,.tox,doc,conf.py
 [testenv:jsonlint]
 commands=
   jsonlint -s next.json
-  jsonlint -s previous_guidelines/2015.07.json
+  jsonlint -s 2015.07.json
-  jsonlint -s previous_guidelines/2016.01.json
+  jsonlint -s 2016.01.json
-  jsonlint -s previous_guidelines/2016.08.json
+  jsonlint -s 2016.08.json
-  jsonlint -s previous_guidelines/2017.01.json
+  jsonlint -s 2017.01.json
-  jsonlint -s previous_guidelines/2017.09.json
+  jsonlint -s 2017.09.json
-  jsonlint -s previous_guidelines/2018.02.json
+  jsonlint -s 2018.02.json
-  jsonlint -s previous_guidelines/2018.11.json
+  jsonlint -s 2018.11.json
-  jsonlint -s previous_guidelines/2019.06.json
+  jsonlint -s 2019.06.json
-  jsonlint -s previous_guidelines/2019.11.json
+  jsonlint -s 2019.11.json
-  jsonlint -s previous_guidelines/2020.06.json
+  jsonlint -s 2020.06.json
-  jsonlint -s previous_guidelines/2020.11.json
+  jsonlint -s 2020.11.json
-  jsonlint -s current_guideline
   jsonlint -s doc/source/schema/1.5.json
   jsonlint -s doc/source/schema/1.6.json
   jsonlint -s doc/source/schema/2.0.json
   jsonlint -s doc/source/schema/next.2.0.json
   jsonlint -s add-ons/dns.next.json
   jsonlint -s add-ons/orchestration.next.json
-  jsonlint -s add-ons/previous_guidelines/dns.2018.02.json
+  jsonlint -s add-ons/dns.2018.02.json
-  jsonlint -s add-ons/previous_guidelines/dns.2019.06.json
+  jsonlint -s add-ons/dns.2019.06.json
-  jsonlint -s add-ons/previous_guidelines/dns.2019.11.json
+  jsonlint -s add-ons/dns.2019.11.json
-  jsonlint -s add-ons/previous_guidelines/dns.2020.06.json
+  jsonlint -s add-ons/dns.2020.06.json
-  jsonlint -s add-ons/previous_guidelines/dns.2020.11.json
+  jsonlint -s add-ons/dns.2020.11.json
-  jsonlint -s add-ons/previous_guidelines/orchestration.2018.02.json
+  jsonlint -s add-ons/orchestration.2018.02.json
-  jsonlint -s add-ons/previous_guidelines/orchestration.2019.06.json
+  jsonlint -s add-ons/orchestration.2019.06.json
-  jsonlint -s add-ons/previous_guidelines/orchestration.2019.11.json
+  jsonlint -s add-ons/orchestration.2019.11.json
-  jsonlint -s add-ons/previous_guidelines/orchestration.2020.06.json
+  jsonlint -s add-ons/orchestration.2020.06.json
-  jsonlint -s add-ons/previous_guidelines/orchestration.2020.11.json
+  jsonlint -s add-ons/orchestration.2020.11.json
-  jsonlint -s add-ons/previous_guidelines/shared_file_system.2020.11.json
   jsonlint -s add-ons/shared_file_system.next.json
-  jsonlint -s add-ons/dns_current_guideline
-  jsonlint -s add-ons/orchestration_current_guideline
-  jsonlint -s add-ons/shared_file_system_current_guideline
 
 [testenv:jsonschema]
 commands=
   jsonschema doc/source/schema/2.0.json -i next.json
-  jsonschema doc/source/schema/1.5.json -i previous_guidelines/2015.07.json
+  jsonschema doc/source/schema/1.5.json -i 2015.07.json
-  jsonschema doc/source/schema/1.5.json -i previous_guidelines/2016.01.json
+  jsonschema doc/source/schema/1.5.json -i 2016.01.json
-  jsonschema doc/source/schema/1.6.json -i previous_guidelines/2016.08.json
+  jsonschema doc/source/schema/1.6.json -i 2016.08.json
-  jsonschema doc/source/schema/1.6.json -i previous_guidelines/2017.01.json
+  jsonschema doc/source/schema/1.6.json -i 2017.01.json
-  jsonschema doc/source/schema/1.6.json -i previous_guidelines/2017.09.json
+  jsonschema doc/source/schema/1.6.json -i 2017.09.json
-  jsonschema doc/source/schema/2.0.json -i previous_guidelines/2018.02.json
+  jsonschema doc/source/schema/2.0.json -i 2018.02.json
-  jsonschema doc/source/schema/2.0.json -i previous_guidelines/2018.11.json
+  jsonschema doc/source/schema/2.0.json -i 2018.11.json
-  jsonschema doc/source/schema/2.0.json -i previous_guidelines/2019.06.json
+  jsonschema doc/source/schema/2.0.json -i 2019.06.json
-  jsonschema doc/source/schema/2.0.json -i previous_guidelines/2019.11.json
+  jsonschema doc/source/schema/2.0.json -i 2019.11.json
-  jsonschema doc/source/schema/2.0.json -i previous_guidelines/2020.06.json
+  jsonschema doc/source/schema/2.0.json -i 2020.06.json
-  jsonschema doc/source/schema/2.0.json -i previous_guidelines/2020.11.json
+  jsonschema doc/source/schema/2.0.json -i 2020.11.json
-  jsonschema doc/source/schema/2.0.json -i current_guideline
   jsonschema doc/source/schema/2.0.json -i doc/source/schema/next.2.0.json
   jsonschema doc/source/schema/2.0.json -i add-ons/dns.next.json
   jsonschema doc/source/schema/2.0.json -i add-ons/orchestration.next.json
-  jsonschema doc/source/schema/2.0.json -i add-ons/previous_guidelines/dns.2018.02.json
+  jsonschema doc/source/schema/2.0.json -i add-ons/dns.2018.02.json
-  jsonschema doc/source/schema/2.0.json -i add-ons/previous_guidelines/dns.2019.06.json
+  jsonschema doc/source/schema/2.0.json -i add-ons/dns.2019.06.json
-  jsonschema doc/source/schema/2.0.json -i add-ons/previous_guidelines/dns.2019.11.json
+  jsonschema doc/source/schema/2.0.json -i add-ons/dns.2019.11.json
-  jsonschema doc/source/schema/2.0.json -i add-ons/previous_guidelines/dns.2020.06.json
+  jsonschema doc/source/schema/2.0.json -i add-ons/dns.2020.06.json
-  jsonschema doc/source/schema/2.0.json -i add-ons/previous_guidelines/dns.2020.11.json
+  jsonschema doc/source/schema/2.0.json -i add-ons/dns.2020.11.json
-  jsonschema doc/source/schema/2.0.json -i add-ons/previous_guidelines/orchestration.2018.02.json
+  jsonschema doc/source/schema/2.0.json -i add-ons/orchestration.2018.02.json
-  jsonschema doc/source/schema/2.0.json -i add-ons/previous_guidelines/orchestration.2019.06.json
+  jsonschema doc/source/schema/2.0.json -i add-ons/orchestration.2019.06.json
-  jsonschema doc/source/schema/2.0.json -i add-ons/previous_guidelines/orchestration.2019.11.json
+  jsonschema doc/source/schema/2.0.json -i add-ons/orchestration.2019.11.json
-  jsonschema doc/source/schema/2.0.json -i add-ons/previous_guidelines/orchestration.2020.06.json
+  jsonschema doc/source/schema/2.0.json -i add-ons/orchestration.2020.06.json
-  jsonschema doc/source/schema/2.0.json -i add-ons/previous_guidelines/orchestration.2020.11.json
+  jsonschema doc/source/schema/2.0.json -i add-ons/orchestration.2020.11.json
-  jsonschema doc/source/schema/2.0.json -i add-ons/previous_guidelines/shared_file_system.2020.11.json
   jsonschema doc/source/schema/2.0.json -i add-ons/shared_file_system.next.json
-  jsonschema doc/source/schema/2.0.json -i add-ons/dns_current_guideline
-  jsonschema doc/source/schema/2.0.json -i add-ons/orchestration_current_guideline
-  jsonschema doc/source/schema/2.0.json -i add-ons/shared_file_system_current_guideline
 
 [testenv:consistency]
 commands=