Merge "Add tests for mysql-fs alarms"

stacklight_tests/helpers/remote_ops.py

@@ -11,6 +11,7 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
+from fuelweb_test import logger
 
 
 def get_all_bridged_interfaces_for_node(remote, excluded_criteria=None):
@@ -108,3 +109,36 @@ def manage_initctl_service(remote, name, operation="restart"):
     """
     remote.check_call("initctl {operation} {service}".format(
         operation=operation, service=name))
+
+
+def fill_up_filesystem(remote, fs, percent, file_name):
+    """Fill filesystem on node.
+
+        :param remote: SSH connection to the node.
+        :type remote: SSHClient
+        :param fs: name of the filesystem to fill up
+        :type fs: str
+        :param percent: amount of space to be filled in percent.
+        :type percent: int
+        :param file_name: name of the created file
+        :type file_name: str
+
+    """
+    logger.info("Filling up {} filesystem to {} percent".format(fs, percent))
+    cmd = (
+        "fallocate -l $(df | grep {} | awk '{{ printf(\"%.0f\\n\", "
+        "1024 * ((($3 + $4) * {} / 100) - $3))}}') {}".format(
+            fs, percent, file_name))
+    remote.check_call(cmd)
+
+
+def clean_filesystem(remote, filename):
+    """Clean space filled by fill_up_filesystem function
+
+        :param remote: SSH connection to the node.
+        :type remote: SSHClient
+        :param filename: name of the file to delete
+        :type filename: str
+    """
+    logger.info("Removing {} file".format(filename))
+    remote.check_call("rm -f {}".format(filename))

stacklight_tests/run_tests.py

@@ -56,6 +56,7 @@ def import_tests():
         test_smoke_bvt)
     from stacklight_tests.lma_infrastructure_alerting import (  # noqa
         test_system)
+    from stacklight_tests.toolchain import test_alarms  # noqa
     from stacklight_tests.toolchain import test_dedicated_environment  # noqa
     from stacklight_tests.toolchain import test_destructive  # noqa
     from stacklight_tests.toolchain import test_detached_plugins  # noqa

stacklight_tests/toolchain/api.py

@@ -15,6 +15,7 @@
 import os
 import time
 
+from devops.helpers import helpers as devops_helpers
 from fuelweb_test import logger
 from fuelweb_test.tests import base_test_case
 from proboscis import asserts
@@ -322,3 +323,21 @@ class ToolchainApi(object):
                                       for hit in output["hits"]["hits"]]))
         self.helpers.check_notifications(notification_list,
                                          cinder_event_types)
+
+    def check_alarms(self, alarm_type, source, hostname, value,
+                     time_interval="now() - 5m"):
+        query = (
+            "select last(value) from {} where time >= {} and source = '{}' "
+            "and hostname = '{}' and value = {}".format(
+                "{}_status".format(alarm_type), time_interval, source,
+                hostname, value))
+
+        def check_result():
+            result = self.INFLUXDB_GRAFANA.do_influxdb_query(
+                query=query).json()["results"][0]
+            return len(result)
+
+        msg = "Alarm with source {} and value {} was not triggered".format(
+            source, value)
+        devops_helpers.wait(check_result, timeout=60 * 5,
+                            interval=10, timeout_msg=msg)

stacklight_tests/toolchain/test_alarms.py

@@ -0,0 +1,77 @@
+#    Copyright 2016 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from fuelweb_test.helpers.decorators import log_snapshot_after_test
+from fuelweb_test import logger
+from proboscis import test
+
+from stacklight_tests.toolchain import api
+
+OKAY_STATUS = 0
+WARNING_STATUS = 1
+CRITICAL_STATUS = 3
+WARNING_PERCENT = 91
+CRITICAL_PERCENT = 96
+
+
+@test(groups=["plugins"])
+class TestToolchainAlarms(api.ToolchainApi):
+    """Class for testing built-in StackLight Collector alarms.
+    """
+
+    def _check_filesystem_alarms(self, nailgun_node, filesystem, source,
+                                 filename, alarm_type="node"):
+        self.check_alarms(
+            alarm_type, source, nailgun_node["hostname"], OKAY_STATUS)
+        with self.fuel_web.get_ssh_for_nailgun_node(nailgun_node) as remote:
+            self.remote_ops.fill_up_filesystem(
+                remote, filesystem, WARNING_PERCENT, filename)
+            logger.info("Checking {}-warning alarm".format(source))
+            self.check_alarms(
+                alarm_type, source, nailgun_node["hostname"], WARNING_STATUS)
+            self.remote_ops.clean_filesystem(remote, filename)
+            self.check_alarms(
+                alarm_type, source, nailgun_node["hostname"], OKAY_STATUS)
+            self.remote_ops.fill_up_filesystem(
+                remote, filesystem, CRITICAL_PERCENT, filename)
+            logger.info("Checking {}-critical alarm".format(source))
+            self.check_alarms(
+                alarm_type, source, nailgun_node["hostname"], CRITICAL_STATUS)
+            self.remote_ops.clean_filesystem(remote, filename)
+            self.check_alarms(
+                alarm_type, source, nailgun_node["hostname"], OKAY_STATUS)
+
+    @test(depends_on_groups=["deploy_toolchain"],
+          groups=["check_mysql_fs_alarms", "toolchain", "alarms"])
+    @log_snapshot_after_test
+    def check_mysql_fs_alarms(self):
+        """Check that mysql-fs-warning and mysql-fs-critical alarms work as
+        expected.
+
+        Scenario:
+            1. Fill up /var/lib/mysql filesystem to 91 percent.
+            2. Check the last value of the warning alarm in InfluxDB.
+            3. Clean the filesystem.
+            4. Fill up /var/lib/mysql filesystem to 96 percent.
+            5. Check the last value of the critical alarm in InfluxDB.
+            6. Clean the filesystem.
+
+        Duration 10m
+        """
+        self.env.revert_snapshot("deploy_toolchain")
+        controller = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
+            self.helpers.cluster_id, ["controller"])[0]
+        self._check_filesystem_alarms(
+            controller, "/dev/mapper/mysql-root", "mysql-fs",
+            "/var/lib/mysql/test/bigfile")