Add API availability tests to upgrade

In order to gauge the impact of upgrades on deployment, we will need to
run tests against the various APIs and gather those results with the
gate.

Fetch the API testing script, start it before upgrades, then look for
the logged results.

Change-Id: I6a5e3d42a39610f8772f9524ca2dfa1942955bc9

scripts/gate-check-commit.sh

@@ -189,11 +189,20 @@ if [[ "${ACTION}" == "upgrade" ]]; then
     unset GROUP_VARS_PATH
     unset HOST_VARS_PATH
 
+    # Fetch script to execute API availability tests, then
+    # background them while the upgrade runs.
+    get_bowling_ball_tests
+    start_bowling_ball_tests
+
     # To execute the upgrade script we need to provide
     # an affirmative response to the warning that the
     # upgrade is irreversable.
     echo 'YES' | bash "$(dirname "${0}")/run-upgrade.sh"
 
+    kill_bowling_ball_tests
+    # Wait to let all the processes finish before looking for output.
+    sleep 10
+    print_bowling_ball_results
 fi
 
 exit_success

scripts/scripts-library.sh

@@ -276,6 +276,36 @@ function get_pip {
   fi
 }
 
+function get_bowling_ball_tests {
+  # Retrieve the latest bowling ball test script in case we don't already have it.
+  if [ -f scripts/rolling_tests.py ]; then
+      return
+  fi
+
+  curl --silent https://raw.githubusercontent.com/openstack/openstack-ansible-ops/master/bowling_ball/rolling_tests.py > scripts/rolling_tests.py
+}
+
+function start_bowling_ball_tests {
+  # The tests will pull Keystone information from the env vars defined in openrc
+  source ~/openrc
+  # Get the list of services to test for from the script, so we're not hard coding.
+  for SERVICE in $(python ./scripts/rolling_tests.py list | cut -d - -f 1); do
+   # Start the scripts in the background and wait between each invocation.
+   # Without the wait, they do not all launch.
+   python ./scripts/rolling_tests.py $SERVICE &
+   sleep 1
+   echo "Started $SERVICE test in background"
+  done
+}
+
+function kill_bowling_ball_tests {
+  pkill -f rolling_tests
+}
+
+function print_bowling_ball_results {
+  grep "failure rate" /var/log/*_rolling.log
+}
+
 ## Signal traps --------------------------------------------------------------
 # Trap all Death Signals and Errors
 trap "exit_fail ${LINENO} $? 'Received STOP Signal'" SIGHUP SIGINT SIGTERM