Make consistency job gating.

This patch updates some of the consistency checking tools
to make them more generally usable by allowing both
automatic checkout of tempest and manual checkout. It builds
on this to add a Zuul job that gates against the consistency
check.

Change-Id: Id0d9148af39c2d2ccf8f11502cc9aff6699f0ead

tools/checktests.py

@@ -152,41 +152,41 @@ def run():
     exit_code = 0
     if len(missing_uuids) > 0:
         exit_code = 1
-        print "### Idempotent ID Errors Detected. To resolve these errors, " \
-              "fix the uuid name (format id-<uuid>) in the guideline to " \
-              "match the id in the test suite:"
+        print("### Idempotent ID Errors Detected. To resolve these errors, "
+              "fix the uuid name (format id-<uuid>) in the guideline to "
+              "match the id in the test suite:")
         for test in missing_uuids:
-            print "Idempotent ID in guideline '%s' does not appear in test " \
-                  "library '%s'\n" \
-                  "  idempotent_id:\n" \
-                  "    %s\n" \
-                  "  names: " % (args.guideline_file, args.testlib, test[0])
+            print("Idempotent ID in guideline '%s' does not appear in test "
+                  "library '%s'\n"
+                  "  idempotent_id:\n"
+                  "    %s\n"
+                  "  names: " % (args.guideline_file, args.testlib, test[0]))
             for testname in test[1]:
-                print "    %s" % (testname)
-            print ""
+                print("    %s" % (testname))
+            print("")
 
     if len(missing_tests) > 0:
         exit_code = 1
-        print "### Test Name Errors Detected. " \
-              "To resolve these errors, update " \
-              "the Interop guideline with the missing " \
-              "test names:"
+        print("### Test Name Errors Detected. "
+              "To resolve these errors, update "
+              "the Interop guideline with the missing "
+              "test names:")
         for uuid in missing_tests:
-            print "Test found in test library '%s'\n" \
-                  "  idempotent_id:\n" \
-                  "    %s\n" \
-                  "  name:\n" \
-                  "    %s\n" \
-                  "Entry in guideline '%s'\n" \
-                  "  idempotent_id:\n" \
-                  "    %s\n" \
+            print("Test found in test library '%s'\n"
+                  "  idempotent_id:\n"
+                  "    %s\n"
+                  "  name:\n"
+                  "    %s\n"
+                  "Entry in guideline '%s'\n"
+                  "  idempotent_id:\n"
+                  "    %s\n"
                   "  names: " % (args.testlib,
                                  uuid, tests[uuid],
                                  args.guideline_file,
-                                 missing_tests[uuid][0])
+                                 missing_tests[uuid][0]))
             for testname in missing_tests[uuid][1]:
-                print "    %s" % (testname)
-            print ""
+                print("    %s" % (testname))
+            print("")
 
     sys.exit(exit_code)
 

tools/consistency.sh

@@ -1,19 +1,40 @@
 #!/bin/bash
 
-tempestdir=$(mktemp -d)
-echo $tempestdir
-git clone git://git.openstack.org/openstack/tempest $tempestdir
+# This script will run consistency checks for Tempest tests against
+# the three latest interoperability guidelines. It can run in two
+# modes.
+#
+# * If no arguments are specified, the script will check out Tempest
+#   into a temporary directory, run the consistency checks, then delete
+#   temporary checkout.
+#
+# * If an argument is given, this script will assume that it is a
+#   user checked-out repository and run the consistency checks against
+#   that, and leave the directory unchanged on exit. This mode is useful
+#   for gate jobs and Tempest development.
 
+set -x
 
-PYTHONPATH=$tempestdir python ./tools/checktests.py --guideline next.json
+if [ ! $@ ]; then
+  TEMPESTDIR=$(mktemp -d)
+  git clone git://git.openstack.org/openstack/tempest $TEMPESTDIR
+  CLEANTEMPEST=cleantempest
+else
+  TEMPESTDIR=${1}
+fi
+
+PYTHONPATH=$TEMPESTDIR python ./tools/checktests.py --guideline next.json
 exit_1=$?
 
-PYTHONPATH=$tempestdir python ./tools/checktests.py --guideline 2018.02.json
+PYTHONPATH=$TEMPESTDIR python ./tools/checktests.py --guideline 2018.02.json
 exit_2=$?
 
-PYTHONPATH=$tempestdir python ./tools/checktests.py --guideline 2017.09.json
+PYTHONPATH=$TEMPESTDIR python ./tools/checktests.py --guideline 2017.09.json
 exit_3=$?
 
-rm -rf $tempestdir
+if [[ ! -z "${CLEANTEMPEST}" ]]; then
+  rm -rf $TEMPESTDIR
+fi
+
 
 ! (( $exit_1 || $exit_2 || $exit_3 ))

tox.ini

@@ -76,7 +76,7 @@ commands=
 
 [testenv:consistency]
 commands=
-  bash tools/consistency.sh
+  bash tools/consistency.sh {posargs}
 whitelist_externals =
   sh
   bash

zuul.d/jobs.yaml

@@ -0,0 +1,12 @@
+- job:
+    name: interop-tempest-consistency
+    parent: tox
+    description: |
+      Ensure consistency between Tempest source and current interop guidelines.
+    vars:
+      tox_envlist: consistency
+      tox_extra_args: "-vv -- {{ ansible_user_dir }}/{{ zuul.projects['git.openstack.org/openstack/tempest'].src_dir }}"
+      zuul_work_dir: src/git.openstack.org/openstack/interop
+    required-projects:
+      - name: openstack/tempest
+      - name: openstack/interop

zuul.d/projects.yaml

@@ -4,6 +4,8 @@
     check:
       jobs:
         - openstack-tox-pep8
+        - interop-tempest-consistency
     gate:
       jobs:
         - openstack-tox-pep8
+        - interop-tempest-consistency