diff --git a/tests/cli_wrapper.py b/tests/cli_wrapper.py
index 2d67e19..1fe28bf 100644
--- a/tests/cli_wrapper.py
+++ b/tests/cli_wrapper.py
@@ -71,7 +71,8 @@ def change_alarm_state(alarm_id, new_state):
     if result_json['state'] != new_state:
         print('Alarm patch failed, expected state of %s but was %s' %
               (result_json['state'], new_state), file=sys.stderr)
-        return 1
+        return False
+    return True
 
 
 def find_alarms_for_definition(alarm_definition_id):
diff --git a/tests/smoke.py b/tests/smoke.py
index b01cebe..893a38c 100755
--- a/tests/smoke.py
+++ b/tests/smoke.py
@@ -206,7 +206,10 @@ def smoke_test(mail_host, metric_host):
 
     new_state = 'OK'
     states.append(new_state)
-    cli_wrapper.change_alarm_state(alarm_id, new_state)
+    if not cli_wrapper.change_alarm_state(alarm_id, new_state):
+        print('Unabled to change Alarm state', file=sys.stderr)
+        return False
+
     # There is a bug in the API which allows this to work. Soon that
     # will be fixed and this will fail
     if len(sys.argv) > 1: