Refactored code into cli_wrapper.py and alarm.py for reuse.

Fixed the notification_cycleTest.py to work on new single node Mini-mon and not re-use random alarms as they weren't always configured properly

Removed dead code from notification_crud.py

tests/alarm.py

@@ -40,8 +40,8 @@ def set_optional_field(name, value, fields):
         fields[name] = value
 
 
-def create(mon_client, name, description, expression, ok_actions,
+def create(mon_client, name, description, expression, ok_actions=None,
-           alarm_actions, undetermined_actions):
+           alarm_actions=None, undetermined_actions=None):
     fields = {}
     fields['name'] = name
     fields['expression'] = expression
@@ -51,3 +51,11 @@ def create(mon_client, name, description, expression, ok_actions,
     set_optional_field('undetermined_actions', undetermined_actions, fields)
     result = mon_client.alarms.create(**fields)
     return result['id']
+
+
+def find_alarm_byname(mon_client, alarm_name):
+    alarms = mon_client.alarms.list(**{})
+    for alarm in alarms:
+        if alarm['name'] == alarm_name:
+            return alarm
+    return None

tests/cli_wrapper.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+#
+""" Wrapper code for running the CLI as a process.
+"""
+from __future__ import print_function
+import sys
+import subprocess
+import json
+
+
+def find_obj_for_name(object_json, name):
+    for obj in object_json:
+        this_name = obj['name']
+        if name == this_name:
+            return obj
+    return None
+
+
+def find_alarm_by_name(alarm_name):
+    alarm_json = run_mon_cli(['alarm-list'])
+    return find_obj_for_name(alarm_json, alarm_name)
+
+
+def delete_alarm_if_exists(alarm_name):
+    alarm_json = find_alarm_by_name(alarm_name)
+    if alarm_json:
+        run_mon_cli(['alarm-delete', alarm_json['id']], useJson=False)
+
+
+def delete_notification_if_exists(notification_name):
+    notification_json = run_mon_cli(['notification-list'])
+    notification = find_obj_for_name(notification_json, notification_name)
+    if notification:
+        run_mon_cli(['notification-delete', notification['id']], useJson=False)
+
+
+def run_mon_cli(args, useJson=True):
+    if useJson:
+        args.insert(0, '--json')
+    args.insert(0, 'mon')
+    try:
+        stdout = subprocess.check_output(args)
+        if useJson:
+            return json.loads(stdout)
+        else:
+            return stdout
+    except subprocess.CalledProcessError as e:
+        print(e, file=sys.stderr)
+        sys.exit(1)
+
+
+def create_notification(notification_name, notification_email_addr):
+    print('Creating notification')
+    result_json = run_mon_cli(['notification-create', notification_name,
+                              'EMAIL', notification_email_addr])
+
+    # Parse out id
+    notification_id = result_json['id']
+    return notification_id
+
+
+def get_alarm_state(alarm_id):
+    result_json = run_mon_cli(['alarm-show', alarm_id])
+    return result_json['state']
+
+
+def patch_alarm(alarm_id, what, value):
+    result_json = run_mon_cli(['alarm-patch', what, value, alarm_id])
+    return result_json
+
+
+def change_alarm_state(alarm_id, new_state):
+    print('Changing Alarm state to %s' % new_state)
+    result_json = patch_alarm(alarm_id, '--state', new_state)
+    if result_json['state'] != new_state:
+        print('Alarm patch failed, expected state of %s but was %s' %
+              (result_json['state'], new_state), file=sys.stderr)
+        return 1
+
+
+def create_alarm(name, expression, description=None, ok_notif_id=None,
+                 alarm_notif_id=None,
+                 undetermined_notif_id=None):
+    args = ['alarm-create']
+    add_argument_if_given(args, '--description', description)
+    add_argument_if_given(args, '--alarm-actions', alarm_notif_id)
+    add_argument_if_given(args, '--ok-actions', ok_notif_id)
+    add_argument_if_given(args, '--undetermined-actions',
+                          undetermined_notif_id)
+    args.append(name)
+    args.append(expression)
+    print('Creating alarm')
+    result_json = run_mon_cli(args)
+
+    # Parse out id
+    alarm_id = result_json['id']
+    return alarm_id
+
+
+def add_argument_if_given(args, arg, value):
+    if value is not None:
+        args.append(arg)
+        args.append(value)

tests/notification_crud.py

@@ -14,18 +14,6 @@ import monclient.exc as exc
 import alarm
 
 
-def call_mon_api(method, fields):
-
-    try:
-        resp = method(**fields)
-    except exc.HTTPException as he:
-        print(he.code)
-        print(he.message)
-        sys.exit(1)
-    else:
-        return resp
-
-
 def cycle_states(mon_client, alarm_id, states):
     print('Cycling alarm states through %s' % (states))
     for state in states:

tests/notification_cycleTest.py

@@ -7,16 +7,9 @@ from __future__ import print_function
 import sys
 import time
 from monclient import client
-from notification import find_notifications
+import notification
 import alarm
-
+import os
-
-def find_alarm_id(mon_client):
-    result = mon_client.alarms.list(**{})
-    if len(result) == 0:
-        print('No existing alarms, create one and rerun test', file=sys.stderr)
-        return None
-    return result[0]['id']
 
 
 def main():
@@ -24,26 +17,46 @@ def main():
         print('usage: %s count [alarm-id]' % sys.argv[0], file=sys.stderr)
         return 1
 
+    if not os.path.isfile('/etc/mon/notification.yaml'):
+        print('Must be run on a VM with Notification Engine installed',
+              file=sys.stderr)
+        return 1
+        
+    # Determine if we are running on mutiple VMs or just the one
+    if os.path.isfile('/etc/mon/mon-api-config.yml'):
+        api_host = 'localhost'
+    else:
+        api_host = '192.168.10.4'
+
     api_version = '2_0'
-    endpoint = 'http://192.168.10.4:8080/v2.0'
+    endpoint = 'http://' + api_host + ':8080/v2.0'
     kwargs = {'token': '82510970543135'}
     global mon_client
     mon_client = client.Client(api_version, endpoint, **kwargs)
 
     num_cycles = int(sys.argv[1])
-    if len(sys.argv) > 2:
+
-        alarm_id = sys.argv[2]
+    alarm_name = 'notification_cycleTest'
+    alarm_json = alarm.find_alarm_byname(mon_client, alarm_name)
+    if alarm_json is not None:
+        alarm_id = alarm_json['id']
     else:
-        alarm_id = find_alarm_id(mon_client)
+        existing = notification.find_by_name(mon_client, alarm_name)
-        if alarm_id is None:
+        if existing is not None:
-            return 1
+            notification_id = existing['id']
+        else:
+            notification_id = notification.create(mon_client, alarm_name,
+                                                  "root@localhost")
+        alarm_id = alarm.create(mon_client, alarm_name, None, 'max(cc) > 100',
+                                notification_id, notification_id,
+                                notification_id)
 
     user = 'root'
     start_time = time.time()
     initial_state = alarm.get_state(mon_client, alarm_id)
     state = initial_state
 
-    existing_notifications = find_notifications(alarm_id, user)
+    existing_notifications = notification.find_notifications(alarm_id, user)
     notifications_sent = num_cycles * 2
     for _ in range(0, notifications_sent):
         if state == 'OK':
@@ -57,7 +70,7 @@ def main():
           ((time.time() - start_time), num_cycles * 2))
 
     for i in range(0, 30):
-        notifications = find_notifications(alarm_id, user)
+        notifications = notification.find_notifications(alarm_id, user)
         notifications_found = len(notifications) - len(existing_notifications)
         if notifications_found >= notifications_sent:
             break

tests/smoke.py

@@ -14,39 +14,24 @@
 """
 
 from __future__ import print_function
-import json
 import sys
 import os
 import subprocess
 import time
+import cli_wrapper
 from notification import find_notifications
-import platform
 
 # export OS_AUTH_TOKEN=82510970543135
 # export OS_NO_CLIENT_AUTH=1
 # export MON_API_URL=http://192.168.10.4:8080/v2.0/
 
 
-def change_alarm_state(alarm_id, new_state):
-    print('Changing Alarm state to %s' % new_state)
-    result_json = run_mon_cli(['alarm-patch', '--state', new_state, alarm_id])
-    if result_json['state'] != new_state:
-        print('Alarm patch failed, expected state of %s but was %s' %
-              (result_json['state'], new_state), file=sys.stderr)
-        return 1
-
-
-def get_alarm_state(alarm_id):
-    result_json = run_mon_cli(['alarm-show', alarm_id])
-    return result_json['state']
-
-
 def check_alarm_history(alarm_id, states):
     transitions = len(states) - 1
     print('Checking Alarm History')
     # May take some time for Alarm history to flow all the way through
     for _ in range(0, 10):
-        result_json = run_mon_cli(['alarm-history', alarm_id])
+        result_json = cli_wrapper.run_mon_cli(['alarm-history', alarm_id])
         if len(result_json) >= transitions:
             break
         time.sleep(4)
@@ -80,27 +65,6 @@ def check_expected(expected, actual, what):
     return False
 
 
-def create_alarm(name, expression, notification_method_id, description=None):
-    args = ['alarm-create']
-    if (description):
-            args.append('--description')
-            args.append(description)
-    args.append('--alarm-actions')
-    args.append(notification_method_id)
-    args.append('--ok-actions')
-    args.append(notification_method_id)
-    args.append('--undetermined-actions')
-    args.append(notification_method_id)
-    args.append(name)
-    args.append(expression)
-    print('Creating alarm')
-    result_json = run_mon_cli(args)
-
-    # Parse out id
-    alarm_id = result_json['id']
-    return alarm_id
-
-
 def get_metrics(name, dimensions):
     print('Getting metrics for %s ' % (name + str(dimensions)))
     dimensions_arg = ''
@@ -108,54 +72,13 @@ def get_metrics(name, dimensions):
         if dimensions_arg != '':
             dimensions_arg = dimensions_arg + ','
         dimensions_arg = dimensions_arg + key + '=' + value
-    return run_mon_cli(['measurement-list', '--dimensions',
+    return cli_wrapper.run_mon_cli(['measurement-list', '--dimensions',
-                       dimensions_arg, name, '00'])
+                                    dimensions_arg, name, '00'])
-
-
-def run_mon_cli(args, useJson=True):
-    if useJson:
-        args.insert(0, '--json')
-    args.insert(0, 'mon')
-    try:
-        stdout = subprocess.check_output(args)
-        if useJson:
-            return json.loads(stdout)
-        else:
-            return stdout
-    except subprocess.CalledProcessError as e:
-        print(e, file=sys.stderr)
-        sys.exit(1)
-
-
-def create_notification(notification_name, notification_email_addr):
-    print('Creating notification')
-    result_json = run_mon_cli(['notification-create', notification_name,
-                              'EMAIL', notification_email_addr])
-
-    # Parse out id
-    notification_method_id = result_json['id']
-    return notification_method_id
-
-
-def find_id_for_name(object_json, name):
-    for obj in object_json:
-        this_name = obj['name']
-        if name == this_name:
-            return obj['id']
-    return None
 
 
 def cleanup(notification_name, alarm_name):
-    # Delete our alarm if it already exists
+    cli_wrapper.delete_alarm_if_exists(alarm_name)
-    alarm_json = run_mon_cli(['alarm-list'])
+    cli_wrapper.delete_notification_if_exists(notification_name)
-    alarm_id = find_id_for_name(alarm_json, alarm_name)
-    if alarm_id:
-        run_mon_cli(['alarm-delete', alarm_id], useJson=False)
-    # Delete our notification if it already exists
-    notification_json = run_mon_cli(['notification-list'])
-    notification_id = find_id_for_name(notification_json, notification_name)
-    if notification_id:
-        run_mon_cli(['notification-delete', notification_id], useJson=False)
 
 
 def wait_for_alarm_state_change(alarm_id, old_state):
@@ -163,7 +86,7 @@ def wait_for_alarm_state_change(alarm_id, old_state):
     print('Waiting for alarm to change state from %s' % old_state)
     for x in range(0, 250):
         time.sleep(1)
-        state = get_alarm_state(alarm_id)
+        state = cli_wrapper.get_alarm_state(alarm_id)
         if state != old_state:
             print('Alarm state changed to %s in %d seconds' % (state, x))
             return state
@@ -242,16 +165,19 @@ def main():
     start_time = time.time()
 
     # Create Notification through CLI
-    notification_method_id = create_notification(notification_name,
+    notification_id = cli_wrapper.create_notification(notification_name,
-                                                 notification_email_addr)
+                                                      notification_email_addr)
     # Create Alarm through CLI
     expression = 'max(cpu_system_perc) > 0 and ' + \
                  'max(load_avg_1_min{hostname=' + metric_host + '}) > 0'
     description = 'System CPU Utilization exceeds 1% and ' + \
                   'Load exceeds 3 per measurement period'
-    alarm_id = create_alarm(alarm_name, expression, notification_method_id,
+    alarm_id = cli_wrapper.create_alarm(alarm_name, expression,
-                            description)
+                                        description=description,
-    state = get_alarm_state(alarm_id)
+                                        ok_notif_id=notification_id,
+                                        alarm_notif_id=notification_id,
+                                        undetermined_notif_id=notification_id)
+    state = cli_wrapper.get_alarm_state(alarm_id)
     # Ensure it is created in the right state
     initial_state = 'UNDETERMINED'
     states = []
@@ -273,7 +199,7 @@ def main():
 
     new_state = 'OK'
     states.append(new_state)
-    change_alarm_state(alarm_id, new_state)
+    cli_wrapper.change_alarm_state(alarm_id, new_state)
     # There is a bug in the API which allows this to work. Soon that
     # will be fixed and this will fail
     if len(sys.argv) > 1:
@@ -291,12 +217,13 @@ def main():
 
     # If the alarm changes state too fast, then there isn't time for the new
     # metric to arrive. Unlikely, but it has been seen
-    ensure_at_least(time.time() - start_time, 30)
+    ensure_at_least(time.time() - start_time, 35)
     change_time = time.time() - start_time
 
     final_num_metrics = count_metrics(metric_name, metric_dimensions)
     if final_num_metrics <= initial_num_metrics:
-        print('No new metrics received', file=sys.stderr)
+        print('No new metrics received in %d seconds' % change_time,
+              file=sys.stderr)
         return 1
     print('Received %d metrics in %d seconds' %
           ((final_num_metrics - initial_num_metrics),  change_time))