diff --git a/stacklight_tests/base_test.py b/stacklight_tests/base_test.py index 39c74d3..c424f95 100644 --- a/stacklight_tests/base_test.py +++ b/stacklight_tests/base_test.py @@ -14,7 +14,10 @@ import abc +from devops.helpers import helpers as devops_helpers +from fuelweb_test import logger from fuelweb_test.tests import base_test_case +import requests import six from stacklight_tests.helpers import checkers @@ -92,3 +95,18 @@ class PluginApi(object): """Check that the plugin works properly. """ pass + + def wait_plugin_online(self, timeout=5 * 60): + """Wait until the plugin will start working properly. + """ + def check_availability(): + try: + self.check_plugin_online() + return True + except (AssertionError, requests.ConnectionError): + return False + + logger.info('Wait a plugin become online') + msg = "Plugin has not become online after a waiting period" + devops_helpers.wait( + check_availability, timeout=timeout, timeout_msg=msg) diff --git a/stacklight_tests/elasticsearch_kibana/test_destructive.py b/stacklight_tests/elasticsearch_kibana/test_destructive.py new file mode 100644 index 0000000..a3afce4 --- /dev/null +++ b/stacklight_tests/elasticsearch_kibana/test_destructive.py @@ -0,0 +1,80 @@ +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from fuelweb_test.helpers.decorators import log_snapshot_after_test +from proboscis import test + +from stacklight_tests.elasticsearch_kibana import api + + +@test(groups=["plugins"]) +class TestDestructiveElasticsearchPlugin(api.ElasticsearchPluginApi): + """Class for testing plugin failover after network disaster.""" + + @test(depends_on_groups=["deploy_ha_elasticsearch_kibana"], + groups=["check_disaster_elasticsearch_kibana", + "elasticsearch_kibana", "destructive", + "check_cluster_outage_elasticsearch_kibana"]) + @log_snapshot_after_test + def check_cluster_outage_elasticsearch_kibana(self): + """Verify that the backends and dashboards recover + after a network outage of the whole Elasticsearch/Kibana cluster. + + Scenario: + 1. Revert the snapshot with 9 deployed nodes in HA configuration + 2. Simulate a network outage of the whole InfluxDB/Grafana cluster + 3. Wait for at least 7 minutes before network recovery + 4. Wait for all services to be back online + 5. Run OSTF + 6. Check that the cluster's state is okay + + Duration 40m + """ + self.env.revert_snapshot("deploy_ha_elasticsearch_kibana") + + self.helpers.emulate_whole_network_disaster( + delay_before_recover=7 * 60) + + self.wait_plugin_online() + + self.helpers.run_ostf() + + @test(depends_on_groups=["deploy_elasticsearch_kibana"], + groups=["check_disaster_elasticsearch_kibana", + "elasticsearch_kibana", "destructive", + "check_node_outage_elasticsearch_kibana"]) + @log_snapshot_after_test + def check_node_outage_elasticsearch_kibana(self): + """Verify that the backends and dashboards recover after + a network outage on a standalone Elasticsearch/Kibana node. + + Scenario: + 1. Revert the snapshot with 3 deployed nodes + 2. Simulate network interruption on the Elasticsearch/Kibana node + 3. Wait for at least 30 seconds before recover network availability + 4. Check that plugin is working + 5. Run OSTF + + Duration 20m + """ + self.env.revert_snapshot("deploy_elasticsearch_kibana") + + with self.fuel_web.get_ssh_for_nailgun_node( + self.helpers.get_master_node_by_role(self.settings.role_name) + ) as remote: + self.remote_ops.simulate_network_interrupt_on_node(remote) + + self.wait_plugin_online() + + self.helpers.run_ostf() diff --git a/stacklight_tests/influxdb_grafana/api.py b/stacklight_tests/influxdb_grafana/api.py index 62f249d..9985b4b 100644 --- a/stacklight_tests/influxdb_grafana/api.py +++ b/stacklight_tests/influxdb_grafana/api.py @@ -12,10 +12,8 @@ # License for the specific language governing permissions and limitations # under the License. -from devops.helpers import helpers as devops_helpers from fuelweb_test import logger from proboscis import asserts -import requests from stacklight_tests import base_test from stacklight_tests.influxdb_grafana.grafana_ui import api as ui_api @@ -105,19 +103,6 @@ class InfluxdbPluginApi(base_test.PluginApi): count, nodes_count_responsed) asserts.assert_equal(count, nodes_count_responsed, msg) - def wait_plugin_online(self, timeout=5 * 60): - def check_availability(): - try: - self.check_plugin_online() - return True - except (AssertionError, requests.ConnectionError): - return False - - logger.info('Wait a plugin become online') - msg = "Plugin has not become online after waiting period" - devops_helpers.wait( - check_availability, timeout=timeout, timeout_msg=msg) - def uninstall_plugin(self): return self.helpers.uninstall_plugin(self.settings.name, self.settings.version) diff --git a/stacklight_tests/influxdb_grafana/test_destructive.py b/stacklight_tests/influxdb_grafana/test_destructive.py index 89e674d..8a0b5de 100644 --- a/stacklight_tests/influxdb_grafana/test_destructive.py +++ b/stacklight_tests/influxdb_grafana/test_destructive.py @@ -47,8 +47,6 @@ class TestDestructiveInfluxdbPlugin(api.InfluxdbPluginApi): self.wait_plugin_online() - self.check_plugin_online() - self.helpers.run_ostf() @test(depends_on_groups=["deploy_influxdb_grafana"], @@ -71,11 +69,10 @@ class TestDestructiveInfluxdbPlugin(api.InfluxdbPluginApi): self.env.revert_snapshot("deploy_influxdb_grafana") with self.fuel_web.get_ssh_for_nailgun_node( - self.get_influxdb_master_node()) as remote: + self.helpers.get_master_node_by_role(self.settings.role_name) + ) as remote: self.remote_ops.simulate_network_interrupt_on_node(remote) self.wait_plugin_online() - self.check_plugin_online() - self.helpers.run_ostf()