More eefactoring for reuse. Added utils.py.

Added alarm_crud.py to test more alarm functionality.

tests/alarm.py

@@ -1,5 +1,9 @@
 from __future__ import print_function
 import sys
+#
+"""
+    Utility methods for CRUD of alarms
+"""
 
 
 def get_state(mon_client, alarm_id):

tests/alarm_crud.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python
+#
+""" Threshold Engine Test
+    CRUD test
+"""
+from __future__ import print_function
+import sys
+import os
+import time
+import cli_wrapper
+import utils
+
+
+def output_metrics(alarm_id, expected_state, metrics):
+    print('Generating metrics, waiting for state change to %s' %
+          expected_state)
+    hostnames = ['AA', 'BB', 'CC']
+    for x in range(0, 90):
+        for metric in metrics:
+            metric_name = metric[0]
+            dimensions = metric[1]
+            args = ['metric-create', '--dimensions']
+            hostname = hostnames[x % len(hostnames)]
+            args.append(dimensions + ',' + 'hostname=' + hostname)
+            args.append(metric_name)
+            args.append('42')
+            cli_wrapper.run_mon_cli(args, useJson=False)
+
+        state = cli_wrapper.get_alarm_state(alarm_id)
+        if state == expected_state:
+            break
+        time.sleep(1)
+
+    if state != expected_state:
+        print('Did not change to state %s instead was %s in %d seconds' %
+              (expected_state, state, x), file=sys.stderr)
+        return False
+
+    print('Changed to state %s in %d seconds' % (state, x))
+    return True
+
+
+def main():
+    utils.setup_cli()
+
+    alarm_name = 'alarm_crud'
+    metric_name = 'alarm_crud'
+    base_dimension = 'service=alarm_test'
+    expression = 'max(%s{%s}) > 0' % (metric_name, base_dimension)
+    description = alarm_name + ' Description'
+    cli_wrapper.delete_alarm_if_exists(alarm_name)
+
+    # Add Alarm
+    alarm_id = cli_wrapper.create_alarm(alarm_name, expression,
+                                        description=description)
+
+    # Ensure it is created in the right state
+    initial_state = 'UNDETERMINED'
+    if not utils.check_alarm_state(alarm_id, initial_state):
+        return 1
+
+    states = []
+    states.append(initial_state)
+
+    # List Alarms, make sure new one shows up
+    alarm_json = cli_wrapper.find_alarm_by_name(alarm_name)
+    if alarm_json is None:
+        print('Did not find alarm named %s using alarm-list' %
+              alarm_name, file=sys.stderr)
+        return 1
+
+    if alarm_id != alarm_json['id']:
+        print('Alarm %s has wrong id, expected %s but was %s' %
+              (alarm_name, alarm_id, alarm_json['id']), file=sys.stderr)
+        return 1
+
+    # Output metrics that will cause it to go ALARM
+    # Wait for it to change to ALARM
+    if not output_metrics(alarm_id, 'ALARM', [[metric_name, base_dimension]]):
+        return 1
+
+    states.append('ALARM')
+
+    # Modify Alarm by adding new expression that will cause it to go OK
+    new_metric_name = 'other_metric'
+    new_dimension = 'dim=42'
+    new_expression = '%s and max(%s{%s}) > 100' % (expression,
+                                                   new_metric_name,
+                                                   new_dimension)
+    alarm_json = cli_wrapper.patch_alarm(alarm_id, '--expression',
+                                         new_expression)
+    if alarm_json['expression'] != new_expression:
+        print('Did not change expression to %s instead was %s' %
+              (new_expression, alarm_json['expression']), file=sys.stderr)
+        return 1
+
+    # Output metrics that will cause it to go OK
+    # Wait for it to change to OK
+    if not output_metrics(alarm_id, 'OK', [[metric_name, base_dimension],
+                                           [new_metric_name, new_dimension]]):
+        return 1
+
+    states.append('OK')
+
+    # Modify Alarm by deleting expression that will cause Alarm to go ALARM
+    cli_wrapper.patch_alarm(alarm_id, '--expression', expression)
+
+    # Output metrics that will cause it to go ALARM
+    # Wait for it to change to ALARM
+    if not output_metrics(alarm_id, 'ALARM',
+                          [[metric_name, base_dimension],
+                           [new_metric_name, new_dimension]]):
+        return 1
+
+    states.append('ALARM')
+
+    # Query History
+    # Delete ALARM
+    cli_wrapper.run_mon_cli(['alarm-delete', alarm_id], useJson=False)
+
+    # Ensure it can't be queried
+    if cli_wrapper.find_alarm_by_name(alarm_name) is not None:
+        print('Still found alarm %s after it was deleted' % alarm_name,
+              file=sys.stderr)
+        return 1
+
+    # Query History, ensure they still show up
+    if not utils.check_alarm_history(alarm_id, states):
+        return 1
+
+    # Success
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/notification_crud.py

@@ -5,13 +5,12 @@
 """
 from __future__ import print_function
 import sys
-import platform
 import os
 import time
-from monclient import client
 import notification
 import monclient.exc as exc
 import alarm
+import utils
 
 
 def cycle_states(mon_client, alarm_id, states):
@@ -70,7 +69,7 @@ def check_notifications(alarm_id, email1, email2, email3, state1, state2,
     return True
 
 
-def print_actions(state, action_ids):
+def print_actions(mon_client, state, action_ids):
     addresses = []
     for action_id in action_ids:
         action_notification = notification.get(mon_client, action_id)
@@ -80,32 +79,21 @@ def print_actions(state, action_ids):
 
 def print_notification_setup(mon_client, alarm_id):
     alarm_data = alarm.get(mon_client, alarm_id)
-    print_actions('ALARM', alarm_data['alarm_actions'])
+    print_actions(mon_client, 'ALARM', alarm_data['alarm_actions'])
-    print_actions('OK', alarm_data['ok_actions'])
+    print_actions(mon_client, 'OK', alarm_data['ok_actions'])
-    print_actions('UNDETERMINED', alarm_data['undetermined_actions'])
+    print_actions(mon_client, 'UNDETERMINED',
+                  alarm_data['undetermined_actions'])
 
 
 def main():
-    hostname = platform.node()
+    if not utils.ensure_has_notification_engine():
-
-    if hostname == 'mini-mon':
-        api_host = '192.168.10.4'
-    elif hostname == 'kafka':
-        api_host = 'localhost'
-    else:
-        print('This test must be run on the kafka or mini-mon VM, aborting',
-              file=sys.stderr)
         return 1
 
     # Delete notification for OK.Cycle OK, ALARM, UNDETERMINED
     # Ensure proper notifications got written for ALARM, UNDETERMINED
 
     states = ['OK', 'ALARM', 'UNDETERMINED']
-    api_version = '2_0'
+    mon_client = utils.create_mon_client()
-    endpoint = 'http://' + api_host + ':8080/v2.0'
-    kwargs = {'token': '82510970543135'}
-    global mon_client
-    mon_client = client.Client(api_version, endpoint, **kwargs)
 
     try:
         # Create 3 notifications with different emails, root, kafka, mon-agent

tests/notification_cycleTest.py

@@ -6,10 +6,9 @@
 from __future__ import print_function
 import sys
 import time
-from monclient import client
 import notification
 import alarm
-import os
+import utils
 
 
 def main():
@@ -17,23 +16,10 @@ def main():
         print('usage: %s count [alarm-id]' % sys.argv[0], file=sys.stderr)
         return 1
 
-    if not os.path.isfile('/etc/mon/notification.yaml'):
+    if not utils.ensure_has_notification_engine():
-        print('Must be run on a VM with Notification Engine installed',
-              file=sys.stderr)
         return 1
-        
-    # Determine if we are running on mutiple VMs or just the one
-    if os.path.isfile('/etc/mon/mon-api-config.yml'):
-        api_host = 'localhost'
-    else:
-        api_host = '192.168.10.4'
-
-    api_version = '2_0'
-    endpoint = 'http://' + api_host + ':8080/v2.0'
-    kwargs = {'token': '82510970543135'}
-    global mon_client
-    mon_client = client.Client(api_version, endpoint, **kwargs)
 
+    mon_client = utils.create_mon_client()
     num_cycles = int(sys.argv[1])
 
     alarm_name = 'notification_cycleTest'

tests/smoke.py

@@ -19,6 +19,7 @@ import os
 import subprocess
 import time
 import cli_wrapper
+import utils
 from notification import find_notifications
 
 # export OS_AUTH_TOKEN=82510970543135
@@ -26,45 +27,6 @@ from notification import find_notifications
 # export MON_API_URL=http://192.168.10.4:8080/v2.0/
 
 
-def check_alarm_history(alarm_id, states):
-    transitions = len(states) - 1
-    print('Checking Alarm History')
-    # May take some time for Alarm history to flow all the way through
-    for _ in range(0, 10):
-        result_json = cli_wrapper.run_mon_cli(['alarm-history', alarm_id])
-        if len(result_json) >= transitions:
-            break
-        time.sleep(4)
-
-    result = True
-    if not check_expected(transitions, len(result_json),
-                          'number of history entries'):
-        return False
-    result_json.sort(key=lambda x: x['timestamp'])
-    for i in range(0, transitions):
-        old_state = states[i]
-        new_state = states[i+1]
-        alarm_json = result_json[i]
-        if not check_expected(old_state, alarm_json['old_state'], 'old_state'):
-            result = False
-        if not check_expected(new_state, alarm_json['new_state'], 'new_state'):
-            result = False
-        if not check_expected(alarm_id, alarm_json['alarm_id'], 'alarm_id'):
-            result = False
-
-    if result:
-        print('Alarm History is OK')
-    return result
-
-
-def check_expected(expected, actual, what):
-    if (expected == actual):
-        return True
-    print("Incorrect value for alarm history %s expected '%s' but was '%s'" %
-          (what, str(expected), str(actual)), file=sys.stderr)
-    return False
-
-
 def get_metrics(name, dimensions):
     print('Getting metrics for %s ' % (name + str(dimensions)))
     dimensions_arg = ''
@@ -135,20 +97,13 @@ def ensure_at_least(desired, actual):
 
 
 def main():
-    # Determine if we are running on mutiple VMs or just the one
+    if not utils.ensure_has_notification_engine():
-    if os.path.isfile('/etc/mon/mon-api-config.yml'):
+        return 1
-        api_host = 'localhost'
-        metric_host = subprocess.check_output(['hostname', '-f']).strip()
-        mail_host = 'localhost'
-    else:
-        api_host = '192.168.10.4'
-        metric_host = 'thresh'
-        mail_host = 'kafka'
 
-    # These need to be set because we are invoking the CLI as a process
+    mail_host = 'localhost'
-    os.environ['OS_AUTH_TOKEN'] = '82510970543135'
+    metric_host = subprocess.check_output(['hostname', '-f']).strip()
-    os.environ['OS_NO_CLIENT_AUTH'] = '1'
+
-    os.environ['MON_API_URL'] = 'http://' + api_host + ':8080/v2.0/'
+    utils.setup_cli()
 
     notification_name = 'Jahmon Smoke Test'
     notification_email_addr = 'root@' + mail_host
@@ -177,14 +132,11 @@ def main():
                                         ok_notif_id=notification_id,
                                         alarm_notif_id=notification_id,
                                         undetermined_notif_id=notification_id)
-    state = cli_wrapper.get_alarm_state(alarm_id)
     # Ensure it is created in the right state
     initial_state = 'UNDETERMINED'
-    states = []
+    if not utils.check_alarm_state(alarm_id, initial_state):
-    if state != initial_state:
-        print('Wrong initial alarm state, expected %s but is %s' %
-              (initial_state, state))
         return 1
+    states = []
     states.append(initial_state)
 
     state = wait_for_alarm_state_change(alarm_id, initial_state)
@@ -227,7 +179,7 @@ def main():
         return 1
     print('Received %d metrics in %d seconds' %
           ((final_num_metrics - initial_num_metrics),  change_time))
-    if not check_alarm_history(alarm_id, states):
+    if not utils.check_alarm_history(alarm_id, states):
         return 1
 
     # Notifications are only sent out for the changes, so omit the first state

tests/utils.py

@@ -0,0 +1,92 @@
+from __future__ import print_function
+import sys
+import time
+import os
+import cli_wrapper
+from monclient import client
+
+"""
+    Utility methods for testing
+"""
+
+
+def check_alarm_history(alarm_id, states):
+    transitions = len(states) - 1
+    print('Checking Alarm History')
+    # May take some time for Alarm history to flow all the way through
+    for _ in range(0, 10):
+        result_json = cli_wrapper.run_mon_cli(['alarm-history', alarm_id])
+        if len(result_json) >= transitions:
+            break
+        time.sleep(4)
+
+    result = True
+    if not check_expected(transitions, len(result_json),
+                          'number of history entries'):
+        return False
+    result_json.sort(key=lambda x: x['timestamp'])
+    for i in range(0, transitions):
+        old_state = states[i]
+        new_state = states[i+1]
+        alarm_json = result_json[i]
+        if not check_expected(old_state, alarm_json['old_state'], 'old_state'):
+            result = False
+        if not check_expected(new_state, alarm_json['new_state'], 'new_state'):
+            result = False
+        if not check_expected(alarm_id, alarm_json['alarm_id'], 'alarm_id'):
+            result = False
+
+    if result:
+        print('Alarm History is OK')
+    return result
+
+
+def check_expected(expected, actual, what):
+    if (expected == actual):
+        return True
+    print("Incorrect value for alarm history %s expected '%s' but was '%s'" %
+          (what, str(expected), str(actual)), file=sys.stderr)
+    return False
+
+
+def check_alarm_state(alarm_id, expected):
+    state = cli_wrapper.get_alarm_state(alarm_id)
+    if state != expected:
+        print('Wrong initial alarm state, expected %s but is %s' %
+              (expected, state))
+        return False
+    return True
+
+
+def get_api_host():
+    # Determine if we are running on multiple VMs or just the one
+    if os.path.isfile('/etc/mon/mon-api-config.yml'):
+        return 'localhost'
+    else:
+        return '192.168.10.4'
+
+
+def setup_cli():
+    api_host = get_api_host()
+
+    # These need to be set because we are invoking the CLI as a process
+    os.environ['OS_AUTH_TOKEN'] = '82510970543135'
+    os.environ['OS_NO_CLIENT_AUTH'] = '1'
+    os.environ['MON_API_URL'] = 'http://' + api_host + ':8080/v2.0/'
+
+
+def create_mon_client():
+    api_host = get_api_host()
+
+    api_version = '2_0'
+    endpoint = 'http://' + api_host + ':8080/v2.0'
+    kwargs = {'token': '82510970543135'}
+    return client.Client(api_version, endpoint, **kwargs)
+
+
+def ensure_has_notification_engine():
+    if not os.path.isfile('/etc/mon/notification.yaml'):
+        print('Must be run on a VM with Notification Engine installed',
+              file=sys.stderr)
+        return False
+    return True