Add running functional scripts for defined backend

This CR allows running "functional" tox job for different backends.
Short description about expected workflow:
1. Gate job run ceilometer/ceilometer/tests/functional/hooks/post_test_hook.py
with <backend> parameter.
2. This script run "tox -efunctional" command with defined variable
CEILOMETER_TEST_BACKEND
3. After in "run-functional-tests.sh" setup script for the backend
and testr are runned.

In this CR running testr with pretty_tox.sh script added because
it allows to use subunit-trace output which developed in tempest-lib
and improve useful of testr output.

Partially implements: blueprint ceilometer-functional-tests
Change-Id: Idb66aca0b46779516db2baec856df8223dbe5c13

ceilometer/tests/functional/hooks/post_test_hook.sh

@@ -37,7 +37,8 @@ sudo chown -R jenkins:stack $CEILOMETER_DIR
 echo "Running ceilometer functional test suite"
 set +e
 
-sudo -E -H -u jenkins tox -efunctional
+# NOTE(ityaptin) Expected a script param which contains a backend name
+CEILOMETER_TEST_BACKEND="$1" sudo -E -H -u jenkins tox -efunctional
 EXIT_CODE=$?
 set -e
 

ceilometer/tests/functional/test_empty.py

@@ -0,0 +1,23 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+""" This test is written to avoid failing running the functional test job
+caused by `subunit-trace -f` with empty testr output
+in ./tools/pretty-tox.sh"""
+
+from oslotest import base
+
+
+class TestEmpty(base.BaseTestCase):
+    def test_empty(self):
+        pass

run-functional-tests.sh

@@ -0,0 +1,12 @@
+#!/bin/bash -x
+set -e
+# Use a mongodb backend by default
+
+
+if [ -z $CEILOMETER_TEST_BACKEND ]; then
+    CEILOMETER_TEST_BACKEND="mongodb"
+fi
+echo $CEILOMETER_TEST_BACKEND
+for backend in $CEILOMETER_TEST_BACKEND; do
+    ./setup-test-env-${backend}.sh ./tools/pretty_tox.sh $*
+done

test-requirements.txt

@@ -31,3 +31,4 @@ testscenarios>=0.4
 testtools>=0.9.36,!=1.2.0
 gabbi>=0.12.0  # Apache-2.0
 requests-aws>=0.1.4    # BSD License (3 clause)
+tempest-lib>=0.5.0

tools/pretty_tox.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -o pipefail
+
+TESTRARGS=$1
+
+# --until-failure is not compatible with --subunit see:
+#
+# https://bugs.launchpad.net/testrepository/+bug/1411804
+#
+# this work around exists until that is addressed
+if [[ "$TESTARGS" =~ "until-failure" ]]; then
+    python setup.py testr --slowest --testr-args="$TESTRARGS"
+else
+    python setup.py testr --slowest --testr-args="--subunit $TESTRARGS" | subunit-trace -f
+fi

tox.ini

@@ -36,14 +36,12 @@ commands =
 commands =
    bash -x {toxinidir}/setup-test-env-elastic.sh python setup.py testr --slowest --testr-args="{posargs}"
 
-# TODO(ityaptin) create a script for running different setup-test-env-*.sh
-# for different backends
 [testenv:functional]
 setenv = VIRTUAL_ENV={envdir}
          EVENTLET_NO_GREENDNS=yes
-         OS_TEST_PATH=ceilometer/tests/functional
+         OS_TEST_PATH=ceilometer/tests/functional/
 commands =
-   bash -x {toxinidir}/setup-test-env-mongodb.sh python setup.py testr --slowest --testr-args="{posargs}"
+   bash -x {toxinidir}/run-functional-tests.sh "{posargs}"
 
 [testenv:py34]
 deps = -r{toxinidir}/requirements.txt