From c443ba7f38876baea4037567b6abeebb5bf2491e Mon Sep 17 00:00:00 2001
From: jkilpatr <jkilpatr@redhat.com>
Date: Tue, 14 Mar 2017 13:45:42 -0400
Subject: [PATCH] Exit with nonzero return code if Elastic indexing fails

At the end of a Browbeat run loop through all the tests and their results
to check if any failed to index, if so set the return code to 1.

This way tests will complete even if indexing fails but scripts and other
software checking the return code will fail out properly instead of allowing
data to be lost silently.

Change-Id: If1a7b2b41df35abbb8a6cc63d6003c1ae36664dd
---
 browbeat.py         | 17 ++++++++++++++---
 lib/PerfKit.py      |  1 +
 lib/Rally.py        |  2 ++
 lib/Shaker.py       |  1 +
 lib/WorkloadBase.py |  4 ++++
 5 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/browbeat.py b/browbeat.py
index 29f4d5357..235a9a207 100755
--- a/browbeat.py
+++ b/browbeat.py
@@ -111,11 +111,22 @@ def main():
         _logger.info("Saved browbeat result summary to {}".format(
             os.path.join(result_dir,time_stamp + '.' + 'report')))
         lib.WorkloadBase.WorkloadBase.print_summary()
+
+        browbeat_rc = 0
         if lib.WorkloadBase.WorkloadBase.failure > 0:
-           _logger.info("Browbeat Finished with Failures, UUID: {}".format(browbeat_uuid))
-           sys.exit(1)
+            browbeat_rc = 1
+        if lib.WorkloadBase.WorkloadBase.index_failures > 0:
+            browbeat_rc = 2
+
+        if browbeat_rc == 1:
+           _logger.info("Browbeat finished with test failures, UUID: {}".format(browbeat_uuid))
+           sys.exit(browbeat_rc)
+        elif browbeat_rc == 2:
+           _logger.info("Browbeat finished with Elasticsearch indexing failures, UUID: {}"
+                        .format(browbeat_uuid))
+           sys.exit(browbeat_rc)
         else:
-           _logger.info("Browbeat Finished Successfully, UUID: {}".format(browbeat_uuid))
+           _logger.info("Browbeat finished successfully, UUID: {}".format(browbeat_uuid))
            sys.exit(0)
 
 if __name__ == '__main__':
diff --git a/lib/PerfKit.py b/lib/PerfKit.py
index 687e9d71b..043a2d66d 100644
--- a/lib/PerfKit.py
+++ b/lib/PerfKit.py
@@ -104,6 +104,7 @@ class PerfKit(WorkloadBase.WorkloadBase):
                     if not self.elastic.index_result(result, test_name, result_dir,
                                                      str(result_count), 'result'):
                         index_success = False
+                        self.update_index_failures()
         else:
             complete_result_json = {'browbeat_scenario': benchmark_config}
             complete_result_json['perfkit_errors'] = self.get_error_details(result_dir)
diff --git a/lib/Rally.py b/lib/Rally.py
index c6cd3f1c0..083a6765c 100644
--- a/lib/Rally.py
+++ b/lib/Rally.py
@@ -351,6 +351,8 @@ class Rally(WorkloadBase.WorkloadBase):
                                             # Start indexing
                                             index_status = self.json_result(
                                                 task_id, scenario_name, run, test_name, result_dir)
+                                            if not index_status:
+                                                self.update_index_failures()
                                             self.get_time_dict(to_time, from_time,
                                                                benchmark[
                                                                    'name'], new_test_name,
diff --git a/lib/Shaker.py b/lib/Shaker.py
index 4b501fa25..174dbf8a4 100644
--- a/lib/Shaker.py
+++ b/lib/Shaker.py
@@ -126,6 +126,7 @@ class Shaker(WorkloadBase.WorkloadBase):
             result = self.elastic.combine_metadata(shaker_stats)
             index_status = self.elastic.index_result(result, test_name, result_dir, _type='error')
             if index_status is False:
+                self.update_index_failures()
                 return False
             else:
                 return True
diff --git a/lib/WorkloadBase.py b/lib/WorkloadBase.py
index c38a59011..191add581 100644
--- a/lib/WorkloadBase.py
+++ b/lib/WorkloadBase.py
@@ -23,6 +23,7 @@ class WorkloadBase(object):
     failure = 0
     total_tests = 0
     total_scenarios = 0
+    index_failures = 0
     browbeat = {}
 
     @abc.abstractmethod
@@ -53,6 +54,9 @@ class WorkloadBase(object):
     def update_total_fail_tests(self):
         WorkloadBase.failure += 1
 
+    def update_index_failures(self):
+        WorkloadBase.index_failures += 1
+
     def workload_logger(self, result_dir, workload):
         base = result_dir.split('/')
         if not os.path.isfile("{}/{}/browbeat-{}-run.log".format(base[0], base[1], workload)):