From c0715bff394d3a74f3bf8ec622b372b66d258854 Mon Sep 17 00:00:00 2001
From: jkilpatr <jkilpatr@redhat.com>
Date: Tue, 14 Feb 2017 11:51:47 -0500
Subject: [PATCH] Basic test based return codes for Browbeat

This adds a very simple check for failed tests before Browbeat's exit
if failed tests are found Browbeat will exit with a return code of one.
This will provide Browbeat CI failure when tests fail without interuppting
ongoing tests if a single one fails.

Change-Id: I5382f684fe03d85692a275dc5c03a136004f34d9
---
 browbeat.py          |  7 ++++++-
 doc/source/usage.rst | 17 +++++++++++++++++
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/browbeat.py b/browbeat.py
index 6b3d475f0..29f4d5357 100755
--- a/browbeat.py
+++ b/browbeat.py
@@ -111,7 +111,12 @@ def main():
         _logger.info("Saved browbeat result summary to {}".format(
             os.path.join(result_dir,time_stamp + '.' + 'report')))
         lib.WorkloadBase.WorkloadBase.print_summary()
-        _logger.info("Browbeat Finished, UUID: {}".format(browbeat_uuid))
+        if lib.WorkloadBase.WorkloadBase.failure > 0:
+           _logger.info("Browbeat Finished with Failures, UUID: {}".format(browbeat_uuid))
+           sys.exit(1)
+        else:
+           _logger.info("Browbeat Finished Successfully, UUID: {}".format(browbeat_uuid))
+           sys.exit(0)
 
 if __name__ == '__main__':
     sys.exit(main())
diff --git a/doc/source/usage.rst b/doc/source/usage.rst
index 64ddbd698..b06f3a088 100644
--- a/doc/source/usage.rst
+++ b/doc/source/usage.rst
@@ -144,6 +144,23 @@ using some simple searches such as:
    shaker_uuid: c918a263-3b0b-409b-8cf8-22dfaeeaf33e AND record.concurrency:1 AND record.test:Bi-Directional
 
 
+Interpreting Browbeat Results
+=============================
+
+By default results for each test will be placed in a timestamped folder `results/` inside your Browbeat folder.
+Each run folder will contain output files from the various workloads and benchmarks that ran during that Browbeat
+run, as well as a report card that summarizes the results of the tests.
+
+Browbeat for the most part tries to restrict itself to running tests, it will only exit with a nonzero return code
+if a workload failed to run. If, for example, Rally where to run but not be able to boot any instances on your cloud
+Browbeat would return with RC 0 without any complaints, only by looking into the Rally results for that Browbeat run
+would you determine that your cloud had a problem that made benchmarking it impossible.
+
+Likewise if Rally manages to run at a snails pace, Browbeat will still exit without complaint. Be aware of this when
+running Browbeat and take the time to either view the contents of the results folder after a run. Or setup Elasticsearch
+and Kibana to view them more easily.
+
+
 Working with Multiple Clouds
 ============================