Reorganize guidelines and improve consistency.sh

Reorganize guidelines into previous_guideline directory
and current_guideline that is softlink to the latest approved guideline.
Cleaned up some tooling that hardwired where guidelines lived.

Change-Id: Ia6be9ca6326718488ee5668df3806da5f76dc456

add-ons/dns_current_guideline

@@ -0,0 +1 @@
+previous_guidelines/dns.2020.11.json

add-ons/orchestration_current_guideline

@@ -0,0 +1 @@
+previous_guidelines/orchestration.2020.11.json

add-ons/shared_file_system_current_guideline

@@ -0,0 +1 @@
+previous_guidelines/shared_file_system.2020.11.json

current_guideline

@@ -0,0 +1 @@
+previous_guidelines/2020.11.json

tools/consistency.sh

@@ -1,40 +1,113 @@
 #!/bin/bash
 
 # This script will run consistency checks for Tempest tests against
-# the three latest interoperability guidelines. It can run in two
+# the current and next interoperability guidelines. It can run in two
 # modes.
 #
-# * If no arguments are specified, the script will check out Tempest
+# * If no arguments are specified, the script will check out Tempest and
-#   into a temporary directory, run the consistency checks, then delete
+#   tempest plugins into a temporary directory, run the consistency checks,
-#   temporary checkout.
+#   then delete temporary checkout.
 #
 # * If an argument is given, this script will assume that it is a
 #   user checked-out repository and run the consistency checks against
 #   that, and leave the directory unchanged on exit. This mode is useful
-#   for gate jobs and Tempest development.
+#   for gate jobs and Tempest/tempest plugin development.
 
 set -x
 
-if [ ! $@ ]; then
+# Prints help
+function usage {
+    SCRIPT_NAME="basename ${BASH_SOURCE[0]}"
+    echo "Usage: ${SCRIPT_NAME} [OPTION]..."
+    echo "Consistency check"
+    echo ""
+    echo "  -h  Print this usage message"
+    echo "  -t  Local Tempest directory"
+    echo "  -d  Local designate-tempest-plugin directory"
+    echo "  -o  Local heat-tempest-plugin directory"
+    echo "  -s  Local manila-tempest-plugin directory"
+    echo "  -c  Set if tempest and plugins directory should be removed after"
+    echo "      the consistency check"
+    exit 1
+}
+
+while getopts t:d:o:s:ch FLAG; do
+    case ${FLAG} in
+        t)
+            TEMPESTDIR=${OPTARG}
+            ;;
+        d)
+            DNSDIR=${OPTARG}
+            ;;
+        o)
+            ORCHESTRATIONDIR=${OPTARG}
+            ;;
+        s)
+            SFSDIR=${OPTARG}
+            ;;
+        c)
+            CLEANTEMPEST=true
+            ;;
+        h)  #show help
+            usage
+            ;;
+        \?) #unrecognized option - show help
+            echo -e \\n"Option -$OPTARG not allowed."
+            usage
+            ;;
+    esac
+done
+
+# check if a local directory was given (for Tempest or one of the plugins),
+# if not, create a temp dir and clone the project there
+if [[ -z $TEMPESTDIR ]]; then
     TEMPESTDIR=$(mktemp -d)
     git clone https://opendev.org/openstack/tempest $TEMPESTDIR
-  CLEANTEMPEST=cleantempest
+fi
-else
+if [[ -z $DNSDIR ]]; then
-  TEMPESTDIR=${1}
+    DNSDIR=$(mktemp -d)
+    git clone https://opendev.org/openstack/designate-tempest-plugin $DNSDIR
+fi
+if [[ -z $ORCHESTRATIONDIR ]]; then
+    ORCHESTRATIONDIR=$(mktemp -d)
+    git clone https://opendev.org/openstack/heat-tempest-plugin $ORCHESTRATIONDIR
+fi
+if [[ -z $SFSDIR ]]; then
+    SFSDIR=$(mktemp -d)
+    git clone https://opendev.org/openstack/manila-tempest-plugin $SFSDIR
 fi
 
-PYTHONPATH=$TEMPESTDIR python ./tools/checktests.py --guideline next.json
+
+export PYTHONPATH=$TEMPESTDIR:$DNSDIR:$ORCHESTRATIONDIR:$SFSDIR
+
+python3 ./tools/checktests.py --guideline next.json
 exit_1=$?
+# TODO(kopecmartin) consistency check is commented out temporarily while we fix
+# inconsistency issues in the follow-up patches
+# python3 ./tools/checktests.py --guideline add-ons/dns.next.json --testlib designate_tempest_plugin
+# exit_2=$?
+# python3 ./tools/checktests.py --guideline add-ons/orchestration.next.json --testlib heat_tempest_plugin
+# exit_3=$?
+# python3 ./tools/checktests.py --guideline add-ons/shared_file_system.next.json --testlib manila_tempest_tests
+# exit_4=$?
 
-PYTHONPATH=$TEMPESTDIR python ./tools/checktests.py --guideline 2018.02.json
+python3 ./tools/checktests.py --guideline current_guideline
-exit_2=$?
+exit_5=$?
+# python3 ./tools/checktests.py --guideline add-ons/dns_current_guideline --testlib designate_tempest_plugin
+# exit_6=$?
+# python3 ./tools/checktests.py --guideline add-ons/orchestration_current_guideline --testlib heat_tempest_plugin
+# exit_7=$?
+# python3 ./tools/checktests.py --guideline add-ons/shared_file_system_current_guideline --testlib manila_tempest_tests
+# exit_8=$?
 
-PYTHONPATH=$TEMPESTDIR python ./tools/checktests.py --guideline 2018.11.json
-exit_3=$?
 
-if [[ ! -z "${CLEANTEMPEST}" ]]; then
+if [[ "${CLEANTEMPEST}" ]]; then
   rm -rf $TEMPESTDIR
+  rm -rf $DNSDIR
+  rm -rf $ORCHESTRATIONDIR
+  rm -rf $SFSDIR
 fi
 
 
-! (( $exit_1 || $exit_2 || $exit_3 ))
+# ! (( $exit_1 || $exit_2 || $exit_3 || $exit_4 || $exit_5 || $exit_6 || $exit_7 || $exit_8 ))
+! (( $exit_1 || $exit_5 ))

tox.ini

@@ -45,63 +45,73 @@ exclude = .venv,.git,.tox,doc,conf.py
 [testenv:jsonlint]
 commands=
   jsonlint -s next.json
-  jsonlint -s 2015.07.json
+  jsonlint -s previous_guidelines/2015.07.json
-  jsonlint -s 2016.01.json
+  jsonlint -s previous_guidelines/2016.01.json
-  jsonlint -s 2016.08.json
+  jsonlint -s previous_guidelines/2016.08.json
-  jsonlint -s 2017.01.json
+  jsonlint -s previous_guidelines/2017.01.json
-  jsonlint -s 2017.09.json
+  jsonlint -s previous_guidelines/2017.09.json
-  jsonlint -s 2018.02.json
+  jsonlint -s previous_guidelines/2018.02.json
-  jsonlint -s 2018.11.json
+  jsonlint -s previous_guidelines/2018.11.json
-  jsonlint -s 2019.06.json
+  jsonlint -s previous_guidelines/2019.06.json
-  jsonlint -s 2019.11.json
+  jsonlint -s previous_guidelines/2019.11.json
-  jsonlint -s 2020.06.json
+  jsonlint -s previous_guidelines/2020.06.json
-  jsonlint -s 2020.11.json
+  jsonlint -s previous_guidelines/2020.11.json
+  jsonlint -s current_guideline
   jsonlint -s doc/source/schema/1.5.json
   jsonlint -s doc/source/schema/1.6.json
   jsonlint -s doc/source/schema/2.0.json
   jsonlint -s doc/source/schema/next.2.0.json
   jsonlint -s add-ons/dns.next.json
   jsonlint -s add-ons/orchestration.next.json
-  jsonlint -s add-ons/dns.2018.02.json
+  jsonlint -s add-ons/previous_guidelines/dns.2018.02.json
-  jsonlint -s add-ons/dns.2019.06.json
+  jsonlint -s add-ons/previous_guidelines/dns.2019.06.json
-  jsonlint -s add-ons/dns.2019.11.json
+  jsonlint -s add-ons/previous_guidelines/dns.2019.11.json
-  jsonlint -s add-ons/dns.2020.06.json
+  jsonlint -s add-ons/previous_guidelines/dns.2020.06.json
-  jsonlint -s add-ons/dns.2020.11.json
+  jsonlint -s add-ons/previous_guidelines/dns.2020.11.json
-  jsonlint -s add-ons/orchestration.2018.02.json
+  jsonlint -s add-ons/previous_guidelines/orchestration.2018.02.json
-  jsonlint -s add-ons/orchestration.2019.06.json
+  jsonlint -s add-ons/previous_guidelines/orchestration.2019.06.json
-  jsonlint -s add-ons/orchestration.2019.11.json
+  jsonlint -s add-ons/previous_guidelines/orchestration.2019.11.json
-  jsonlint -s add-ons/orchestration.2020.06.json
+  jsonlint -s add-ons/previous_guidelines/orchestration.2020.06.json
-  jsonlint -s add-ons/orchestration.2020.11.json
+  jsonlint -s add-ons/previous_guidelines/orchestration.2020.11.json
+  jsonlint -s add-ons/previous_guidelines/shared_file_system.2020.11.json
   jsonlint -s add-ons/shared_file_system.next.json
+  jsonlint -s add-ons/dns_current_guideline
+  jsonlint -s add-ons/orchestration_current_guideline
+  jsonlint -s add-ons/shared_file_system_current_guideline
 
 [testenv:jsonschema]
 commands=
   jsonschema doc/source/schema/2.0.json -i next.json
-  jsonschema doc/source/schema/1.5.json -i 2015.07.json
+  jsonschema doc/source/schema/1.5.json -i previous_guidelines/2015.07.json
-  jsonschema doc/source/schema/1.5.json -i 2016.01.json
+  jsonschema doc/source/schema/1.5.json -i previous_guidelines/2016.01.json
-  jsonschema doc/source/schema/1.6.json -i 2016.08.json
+  jsonschema doc/source/schema/1.6.json -i previous_guidelines/2016.08.json
-  jsonschema doc/source/schema/1.6.json -i 2017.01.json
+  jsonschema doc/source/schema/1.6.json -i previous_guidelines/2017.01.json
-  jsonschema doc/source/schema/1.6.json -i 2017.09.json
+  jsonschema doc/source/schema/1.6.json -i previous_guidelines/2017.09.json
-  jsonschema doc/source/schema/2.0.json -i 2018.02.json
+  jsonschema doc/source/schema/2.0.json -i previous_guidelines/2018.02.json
-  jsonschema doc/source/schema/2.0.json -i 2018.11.json
+  jsonschema doc/source/schema/2.0.json -i previous_guidelines/2018.11.json
-  jsonschema doc/source/schema/2.0.json -i 2019.06.json
+  jsonschema doc/source/schema/2.0.json -i previous_guidelines/2019.06.json
-  jsonschema doc/source/schema/2.0.json -i 2019.11.json
+  jsonschema doc/source/schema/2.0.json -i previous_guidelines/2019.11.json
-  jsonschema doc/source/schema/2.0.json -i 2020.06.json
+  jsonschema doc/source/schema/2.0.json -i previous_guidelines/2020.06.json
-  jsonschema doc/source/schema/2.0.json -i 2020.11.json
+  jsonschema doc/source/schema/2.0.json -i previous_guidelines/2020.11.json
+  jsonschema doc/source/schema/2.0.json -i current_guideline
   jsonschema doc/source/schema/2.0.json -i doc/source/schema/next.2.0.json
   jsonschema doc/source/schema/2.0.json -i add-ons/dns.next.json
   jsonschema doc/source/schema/2.0.json -i add-ons/orchestration.next.json
-  jsonschema doc/source/schema/2.0.json -i add-ons/dns.2018.02.json
+  jsonschema doc/source/schema/2.0.json -i add-ons/previous_guidelines/dns.2018.02.json
-  jsonschema doc/source/schema/2.0.json -i add-ons/dns.2019.06.json
+  jsonschema doc/source/schema/2.0.json -i add-ons/previous_guidelines/dns.2019.06.json
-  jsonschema doc/source/schema/2.0.json -i add-ons/dns.2019.11.json
+  jsonschema doc/source/schema/2.0.json -i add-ons/previous_guidelines/dns.2019.11.json
-  jsonschema doc/source/schema/2.0.json -i add-ons/dns.2020.06.json
+  jsonschema doc/source/schema/2.0.json -i add-ons/previous_guidelines/dns.2020.06.json
-  jsonschema doc/source/schema/2.0.json -i add-ons/dns.2020.11.json
+  jsonschema doc/source/schema/2.0.json -i add-ons/previous_guidelines/dns.2020.11.json
-  jsonschema doc/source/schema/2.0.json -i add-ons/orchestration.2018.02.json
+  jsonschema doc/source/schema/2.0.json -i add-ons/previous_guidelines/orchestration.2018.02.json
-  jsonschema doc/source/schema/2.0.json -i add-ons/orchestration.2019.06.json
+  jsonschema doc/source/schema/2.0.json -i add-ons/previous_guidelines/orchestration.2019.06.json
-  jsonschema doc/source/schema/2.0.json -i add-ons/orchestration.2019.11.json
+  jsonschema doc/source/schema/2.0.json -i add-ons/previous_guidelines/orchestration.2019.11.json
-  jsonschema doc/source/schema/2.0.json -i add-ons/orchestration.2020.06.json
+  jsonschema doc/source/schema/2.0.json -i add-ons/previous_guidelines/orchestration.2020.06.json
-  jsonschema doc/source/schema/2.0.json -i add-ons/orchestration.2020.11.json
+  jsonschema doc/source/schema/2.0.json -i add-ons/previous_guidelines/orchestration.2020.11.json
+  jsonschema doc/source/schema/2.0.json -i add-ons/previous_guidelines/shared_file_system.2020.11.json
   jsonschema doc/source/schema/2.0.json -i add-ons/shared_file_system.next.json
+  jsonschema doc/source/schema/2.0.json -i add-ons/dns_current_guideline
+  jsonschema doc/source/schema/2.0.json -i add-ons/orchestration_current_guideline
+  jsonschema doc/source/schema/2.0.json -i add-ons/shared_file_system_current_guideline
 
 [testenv:consistency]
 commands=