x/stacklight-integration-tests
Code Issues Proposed changes

Merge "Add tests for mysql-fs alarms"

Browse Source
This commit is contained in:
Jenkins 2016-08-30 07:27:30 +00:00 committed by Gerrit Code Review
commit 70fe115643
4 changed files with 131 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
stacklight_tests/helpers/remote_ops.py
View File

@ -11,6 +11,7 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from fuelweb_test import logger
def get_all_bridged_interfaces_for_node(remote, excluded_criteria=None):
@ -108,3 +109,36 @@ def manage_initctl_service(remote, name, operation="restart"):
"""
remote.check_call("initctl {operation} {service}".format(
operation=operation, service=name))
def fill_up_filesystem(remote, fs, percent, file_name):
"""Fill filesystem on node.
:param remote: SSH connection to the node.
:type remote: SSHClient
:param fs: name of the filesystem to fill up
:type fs: str
:param percent: amount of space to be filled in percent.
:type percent: int
:param file_name: name of the created file
:type file_name: str
"""
logger.info("Filling up {} filesystem to {} percent".format(fs, percent))
cmd = (
"fallocate -l $(df | grep {} | awk '{{ printf(\"%.0f\\n\", "
"1024 * ((($3 + $4) * {} / 100) - $3))}}') {}".format(
fs, percent, file_name))
remote.check_call(cmd)
def clean_filesystem(remote, filename):
"""Clean space filled by fill_up_filesystem function
:param remote: SSH connection to the node.
:type remote: SSHClient
:param filename: name of the file to delete
:type filename: str
"""
logger.info("Removing {} file".format(filename))
remote.check_call("rm -f {}".format(filename))

1
stacklight_tests/run_tests.py
View File

@ -56,6 +56,7 @@ def import_tests():
test_smoke_bvt)
from stacklight_tests.lma_infrastructure_alerting import ( # noqa
test_system)
from stacklight_tests.toolchain import test_alarms # noqa
from stacklight_tests.toolchain import test_dedicated_environment # noqa
from stacklight_tests.toolchain import test_destructive # noqa
from stacklight_tests.toolchain import test_detached_plugins # noqa

19
stacklight_tests/toolchain/api.py
View File

@ -15,6 +15,7 @@
import os
import time
from devops.helpers import helpers as devops_helpers
from fuelweb_test import logger
from fuelweb_test.tests import base_test_case
from proboscis import asserts
@ -322,3 +323,21 @@ class ToolchainApi(object):
for hit in output["hits"]["hits"]]))
self.helpers.check_notifications(notification_list,
cinder_event_types)
def check_alarms(self, alarm_type, source, hostname, value,
time_interval="now() - 5m"):
query = (
"select last(value) from {} where time >= {} and source = '{}' "
"and hostname = '{}' and value = {}".format(
"{}_status".format(alarm_type), time_interval, source,
hostname, value))
def check_result():
result = self.INFLUXDB_GRAFANA.do_influxdb_query(
query=query).json()["results"][0]
return len(result)
msg = "Alarm with source {} and value {} was not triggered".format(
source, value)
devops_helpers.wait(check_result, timeout=60 * 5,
interval=10, timeout_msg=msg)

77
stacklight_tests/toolchain/test_alarms.py Normal file
View File

@ -0,0 +1,77 @@
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from fuelweb_test.helpers.decorators import log_snapshot_after_test
from fuelweb_test import logger
from proboscis import test
from stacklight_tests.toolchain import api
OKAY_STATUS = 0
WARNING_STATUS = 1
CRITICAL_STATUS = 3
WARNING_PERCENT = 91
CRITICAL_PERCENT = 96
@test(groups=["plugins"])
class TestToolchainAlarms(api.ToolchainApi):
"""Class for testing built-in StackLight Collector alarms.
"""
def _check_filesystem_alarms(self, nailgun_node, filesystem, source,
filename, alarm_type="node"):
self.check_alarms(
alarm_type, source, nailgun_node["hostname"], OKAY_STATUS)
with self.fuel_web.get_ssh_for_nailgun_node(nailgun_node) as remote:
self.remote_ops.fill_up_filesystem(
remote, filesystem, WARNING_PERCENT, filename)
logger.info("Checking {}-warning alarm".format(source))
self.check_alarms(
alarm_type, source, nailgun_node["hostname"], WARNING_STATUS)
self.remote_ops.clean_filesystem(remote, filename)
self.check_alarms(
alarm_type, source, nailgun_node["hostname"], OKAY_STATUS)
self.remote_ops.fill_up_filesystem(
remote, filesystem, CRITICAL_PERCENT, filename)
logger.info("Checking {}-critical alarm".format(source))
self.check_alarms(
alarm_type, source, nailgun_node["hostname"], CRITICAL_STATUS)
self.remote_ops.clean_filesystem(remote, filename)
self.check_alarms(
alarm_type, source, nailgun_node["hostname"], OKAY_STATUS)
@test(depends_on_groups=["deploy_toolchain"],
groups=["check_mysql_fs_alarms", "toolchain", "alarms"])
@log_snapshot_after_test
def check_mysql_fs_alarms(self):
"""Check that mysql-fs-warning and mysql-fs-critical alarms work as
expected.
Scenario:
1. Fill up /var/lib/mysql filesystem to 91 percent.
2. Check the last value of the warning alarm in InfluxDB.
3. Clean the filesystem.
4. Fill up /var/lib/mysql filesystem to 96 percent.
5. Check the last value of the critical alarm in InfluxDB.
6. Clean the filesystem.
Duration 10m
"""
self.env.revert_snapshot("deploy_toolchain")
controller = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
self.helpers.cluster_id, ["controller"])[0]
self._check_filesystem_alarms(
controller, "/dev/mapper/mysql-root", "mysql-fs",
"/var/lib/mysql/test/bigfile")