Add timeout for tests execution

Individual test should not take too much time, but we can "catch" some bad
cases when one test execution took ~30 minute. It means that test was written
in wrong way or there is some external factors. To do not waste too much time,
let's terminate tests if they take more that 1 minute.

Also, this patch moves launch of coverage job to parallel mode

Change-Id: I428998c7e02fee8354a67b2d277c65a743635848

test-requirements.txt

@@ -2,14 +2,18 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 
-# [do-not-touch] we need to modify our code to support latest release of this
+# [do-not-touch] we need to modify our code to support latest release
-# lib
 hacking>=0.9.2,<0.10                                   # Apache Software License
 
 pytest>=2.7                                            # MIT
+# py.test plugin for measuring coverage.
 pytest-cov>=2.2.1                                      # MIT
+# py.test plugin for generating HTML reports
 pytest-html>=1.10.0                                    # Mozilla Public License 2.0 (MPL 2.0)
+# py.test xdist plugin for distributed testing and loop-on-failing modes
 pytest-xdist                                           # MIT
+# py.test plugin to abort hanging tests
+pytest-timeout                                         # MIT
 
 coverage>=3.6                                          # Apache License, Version 2.0
 ddt>=1.0.1

tests/ci/cover.sh

@@ -33,7 +33,7 @@ fi
 git checkout HEAD^
 
 baseline_report=$(mktemp -t rally_coverageXXXXXXX)
-py.test --cov=rally tests/unit/ --cov-report=html
+py.test --cov=rally tests/unit/ --cov-report=html --timeout=60 -n auto
 coverage report > $baseline_report
 mv cover cover-master
 cat $baseline_report
@@ -43,7 +43,7 @@ baseline_missing=$(awk 'END { print $3 }' $baseline_report)
 git checkout -
 
 current_report=$(mktemp -t rally_coverageXXXXXXX)
-py.test --cov=rally tests/unit/ --cov-report=html
+py.test --cov=rally tests/unit/ --cov-report=html --timeout=60 -n auto
 coverage report > $current_report
 current_missing=$(awk 'END { print $3 }' $current_report)
 

tests/ci/pytest_launcher.py

@@ -20,6 +20,13 @@ import sys
 PYTEST_REPORT = os.environ.get("PYTEST_REPORT",
                                ".test_results/pytest_results.html")
 TESTR_REPORT = "testr_results.html"
+PYTEST_ARGUMENTS = ("py.test"  # base command
+                    " --html=%(html_report)s"  # html report
+                    " --durations=10"  # get a list of the slowest 10 tests
+                    " -n auto"  # launch tests in parallel
+                    " --timeout=%(timeout)s"  # timeout for individual test
+                    " %(path)s"
+                    )
 
 
 def error(msg):
@@ -34,6 +41,9 @@ def main(args):
     parser.add_argument("--posargs", metavar="<str>", type=str, default="",
                         help="TOX posargs. Currently supported only string to "
                              "partial test or tests group to launch.")
+    parser.add_argument("--timeout", metavar="<seconds>", type=int, default=60,
+                        help="Timeout for individual test execution. "
+                             "Defaults to 60")
     args = parser.parse_args(args[1:])
 
     # We allow only one parameter - path to partial test or tests group
@@ -86,9 +96,11 @@ def main(args):
     else:
         pytest_report = PYTEST_REPORT
 
+    args = PYTEST_ARGUMENTS % {"html_report": pytest_report,
+                               "path": path,
+                               "timeout": args.timeout}
     try:
-        subprocess.check_call(["py.test", "--html=%s" % pytest_report,
+        subprocess.check_call(args.split(" "),
-                              "--durations=10", "-n", "auto", path],
                               stderr=subprocess.STDOUT)
     except subprocess.CalledProcessError:
         # NOTE(andreykurilin): it is ok, since tests can fail.

tox.ini

@@ -55,7 +55,7 @@ commands = oslo_debug_helper -t tests {posargs}
 sitepackages = True
 commands =
   find . -type f -name "*.pyc" -delete
-  python {toxinidir}/tests/ci/pytest_launcher.py "tests/functional" --posargs={posargs}
+  python {toxinidir}/tests/ci/pytest_launcher.py "tests/functional" --timeout -1 --posargs={posargs}
 
 [testenv:cover]
 commands = {toxinidir}/tests/ci/cover.sh {posargs}