diff --git a/stacklight_tests/helpers/remote_ops.py b/stacklight_tests/helpers/remote_ops.py index 7f74c80..fb50960 100644 --- a/stacklight_tests/helpers/remote_ops.py +++ b/stacklight_tests/helpers/remote_ops.py @@ -11,6 +11,7 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +from fuelweb_test import logger def get_all_bridged_interfaces_for_node(remote, excluded_criteria=None): @@ -108,3 +109,36 @@ def manage_initctl_service(remote, name, operation="restart"): """ remote.check_call("initctl {operation} {service}".format( operation=operation, service=name)) + + +def fill_up_filesystem(remote, fs, percent, file_name): + """Fill filesystem on node. + + :param remote: SSH connection to the node. + :type remote: SSHClient + :param fs: name of the filesystem to fill up + :type fs: str + :param percent: amount of space to be filled in percent. + :type percent: int + :param file_name: name of the created file + :type file_name: str + + """ + logger.info("Filling up {} filesystem to {} percent".format(fs, percent)) + cmd = ( + "fallocate -l $(df | grep {} | awk '{{ printf(\"%.0f\\n\", " + "1024 * ((($3 + $4) * {} / 100) - $3))}}') {}".format( + fs, percent, file_name)) + remote.check_call(cmd) + + +def clean_filesystem(remote, filename): + """Clean space filled by fill_up_filesystem function + + :param remote: SSH connection to the node. + :type remote: SSHClient + :param filename: name of the file to delete + :type filename: str + """ + logger.info("Removing {} file".format(filename)) + remote.check_call("rm -f {}".format(filename)) diff --git a/stacklight_tests/run_tests.py b/stacklight_tests/run_tests.py index 9967dfa..f209854 100644 --- a/stacklight_tests/run_tests.py +++ b/stacklight_tests/run_tests.py @@ -56,6 +56,7 @@ def import_tests(): test_smoke_bvt) from stacklight_tests.lma_infrastructure_alerting import ( # noqa test_system) + from stacklight_tests.toolchain import test_alarms # noqa from stacklight_tests.toolchain import test_dedicated_environment # noqa from stacklight_tests.toolchain import test_destructive # noqa from stacklight_tests.toolchain import test_detached_plugins # noqa diff --git a/stacklight_tests/toolchain/api.py b/stacklight_tests/toolchain/api.py index 4155d08..f4f67a9 100644 --- a/stacklight_tests/toolchain/api.py +++ b/stacklight_tests/toolchain/api.py @@ -15,6 +15,7 @@ import os import time +from devops.helpers import helpers as devops_helpers from fuelweb_test import logger from fuelweb_test.tests import base_test_case from proboscis import asserts @@ -322,3 +323,21 @@ class ToolchainApi(object): for hit in output["hits"]["hits"]])) self.helpers.check_notifications(notification_list, cinder_event_types) + + def check_alarms(self, alarm_type, source, hostname, value, + time_interval="now() - 5m"): + query = ( + "select last(value) from {} where time >= {} and source = '{}' " + "and hostname = '{}' and value = {}".format( + "{}_status".format(alarm_type), time_interval, source, + hostname, value)) + + def check_result(): + result = self.INFLUXDB_GRAFANA.do_influxdb_query( + query=query).json()["results"][0] + return len(result) + + msg = "Alarm with source {} and value {} was not triggered".format( + source, value) + devops_helpers.wait(check_result, timeout=60 * 5, + interval=10, timeout_msg=msg) diff --git a/stacklight_tests/toolchain/test_alarms.py b/stacklight_tests/toolchain/test_alarms.py new file mode 100644 index 0000000..8e3899b --- /dev/null +++ b/stacklight_tests/toolchain/test_alarms.py @@ -0,0 +1,77 @@ +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from fuelweb_test.helpers.decorators import log_snapshot_after_test +from fuelweb_test import logger +from proboscis import test + +from stacklight_tests.toolchain import api + +OKAY_STATUS = 0 +WARNING_STATUS = 1 +CRITICAL_STATUS = 3 +WARNING_PERCENT = 91 +CRITICAL_PERCENT = 96 + + +@test(groups=["plugins"]) +class TestToolchainAlarms(api.ToolchainApi): + """Class for testing built-in StackLight Collector alarms. + """ + + def _check_filesystem_alarms(self, nailgun_node, filesystem, source, + filename, alarm_type="node"): + self.check_alarms( + alarm_type, source, nailgun_node["hostname"], OKAY_STATUS) + with self.fuel_web.get_ssh_for_nailgun_node(nailgun_node) as remote: + self.remote_ops.fill_up_filesystem( + remote, filesystem, WARNING_PERCENT, filename) + logger.info("Checking {}-warning alarm".format(source)) + self.check_alarms( + alarm_type, source, nailgun_node["hostname"], WARNING_STATUS) + self.remote_ops.clean_filesystem(remote, filename) + self.check_alarms( + alarm_type, source, nailgun_node["hostname"], OKAY_STATUS) + self.remote_ops.fill_up_filesystem( + remote, filesystem, CRITICAL_PERCENT, filename) + logger.info("Checking {}-critical alarm".format(source)) + self.check_alarms( + alarm_type, source, nailgun_node["hostname"], CRITICAL_STATUS) + self.remote_ops.clean_filesystem(remote, filename) + self.check_alarms( + alarm_type, source, nailgun_node["hostname"], OKAY_STATUS) + + @test(depends_on_groups=["deploy_toolchain"], + groups=["check_mysql_fs_alarms", "toolchain", "alarms"]) + @log_snapshot_after_test + def check_mysql_fs_alarms(self): + """Check that mysql-fs-warning and mysql-fs-critical alarms work as + expected. + + Scenario: + 1. Fill up /var/lib/mysql filesystem to 91 percent. + 2. Check the last value of the warning alarm in InfluxDB. + 3. Clean the filesystem. + 4. Fill up /var/lib/mysql filesystem to 96 percent. + 5. Check the last value of the critical alarm in InfluxDB. + 6. Clean the filesystem. + + Duration 10m + """ + self.env.revert_snapshot("deploy_toolchain") + controller = self.fuel_web.get_nailgun_cluster_nodes_by_roles( + self.helpers.cluster_id, ["controller"])[0] + self._check_filesystem_alarms( + controller, "/dev/mapper/mysql-root", "mysql-fs", + "/var/lib/mysql/test/bigfile")