HTML test report for Deckhand functional tests

This PS leverages pytest-html to generate an HTML report for
Deckhand functional tests. The test_gabbi.py file was updated to
use pytest as the test runner as it is compatible with pytest-html.
The report is saved in results/index.html. Test docs were updated.

Change-Id: I0b611bf124bf87d801ab93dd2a5d16f136e4801d

.gitignore

@@ -47,6 +47,7 @@ coverage.xml
 .hypothesis/
 .testrepository/*
 cover/*
+results/*
 
 # Translations
 *.mo

deckhand/tests/functional/test_gabbi.py

@@ -15,16 +15,16 @@
 import os
 import yaml
 
-import gabbi.driver
-import gabbi.handlers.jsonhandler
-import gabbi.json_parser
+from gabbi import driver
+from gabbi.driver import test_pytest  # noqa
+from gabbi.handlers import jsonhandler
 
 TESTS_DIR = 'gabbits'
 
 
 # This is quite similar to the existing JSONHandler, so use it as the base
-# class instead of gabbi.handlers.base.ContentHandler
-class MultidocJsonpaths(gabbi.handlers.jsonhandler.JSONHandler):
+# class instead of `gabbi.handlers.base.ContentHandler`.
+class MultidocJsonpaths(jsonhandler.JSONHandler):
     test_key_suffix = 'multidoc_jsonpaths'
 
     @staticmethod
@@ -46,14 +46,15 @@ class MultidocJsonpaths(gabbi.handlers.jsonhandler.JSONHandler):
         return list(yaml.safe_load_all(string))
 
 
-def load_tests(loader, tests, pattern):
+def pytest_generate_tests(metafunc):
     test_dir = os.path.join(os.path.dirname(__file__), TESTS_DIR)
-    return gabbi.driver.build_tests(test_dir, loader,
-        # NOTE(fmontei): When there are multiple handlers listed that
-        # accept the same content-type, the one that is earliest in the
-        # list will be used. Thus, we cannot specify multiple content
-        # handlers for handling list/dictionary responses from the server
-        # using different handlers.
-        content_handlers=[MultidocJsonpaths],
-        verbose=True,
-        url=os.environ['DECKHAND_TEST_URL'])
+    # NOTE(fmontei): While only `url` or `host` is needed, strangely both
+    # are needed because we use `pytest-html` which throws an error without
+    # `host`.
+    driver.py_test_generator(
+        test_dir, url=os.environ['DECKHAND_TEST_URL'], host='localhost',
+        # NOTE(fmontei): When there are multiple handlers listed that accept
+        # the same content-type, the one that is earliest in the list will be
+        # used. Thus, we cannot specify multiple content handlers for handling
+        # list/dictionary responses from the server using different handlers.
+        content_handlers=[MultidocJsonpaths], metafunc=metafunc)

doc/source/testing.rst

@@ -90,6 +90,8 @@ The command executes ``tools/functional-tests.sh`` which:
        Keystone and pass an admin token to Deckhand.
     4) Instantiates Deckhand via ``uwisgi``.
     5) Calls gabbi which runs a battery of functional tests.
+    6) An HTML report that visualizes the result of the test run is output to
+       ``results/index.html``.
 
 At this time, there are no functional tests for policy enforcement
 verification. Negative tests will be added at a later date to confirm that

test-requirements.txt

@@ -19,3 +19,4 @@ gabbi==1.35.1
 sphinx_rtd_theme==0.2.4
 pifpaf==0.10.0
 reno>=2.5.0 # Apache-2.0
+pytest-html==1.16.0

tools/functional-tests.sh

@@ -127,12 +127,14 @@ sleep 5
 
 log_section Running tests
 
+ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
 set +e
 posargs=$@
 if [ ${#posargs} -ge 1 ]; then
-    ostestr --concurrency 1 --regex $1
+    py.test -k $1 -svx $( dirname $ROOTDIR )/deckhand/tests/functional/test_gabbi.py --html=results/index.html
 else
-    ostestr --concurrency 1
+    py.test -svx $( dirname $ROOTDIR )/deckhand/tests/functional/test_gabbi.py --html=results/index.html
 fi
 TEST_STATUS=$?
 set -e