Add failover tests for Elasticsearch

Change-Id: I98d7b437bdfdae7d65fb9bcb0d2b6814f872d2e1

stacklight_tests/base_test.py

@@ -14,7 +14,10 @@
 
 import abc
 
+from devops.helpers import helpers as devops_helpers
+from fuelweb_test import logger
 from fuelweb_test.tests import base_test_case
+import requests
 import six
 
 from stacklight_tests.helpers import checkers
@@ -92,3 +95,18 @@ class PluginApi(object):
         """Check that the plugin works properly.
         """
         pass
+
+    def wait_plugin_online(self, timeout=5 * 60):
+        """Wait until the plugin will start working properly.
+        """
+        def check_availability():
+            try:
+                self.check_plugin_online()
+                return True
+            except (AssertionError, requests.ConnectionError):
+                return False
+
+        logger.info('Wait a plugin become online')
+        msg = "Plugin has not become online after a waiting period"
+        devops_helpers.wait(
+            check_availability, timeout=timeout, timeout_msg=msg)

stacklight_tests/elasticsearch_kibana/test_destructive.py

@@ -0,0 +1,80 @@
+#    Copyright 2016 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from fuelweb_test.helpers.decorators import log_snapshot_after_test
+from proboscis import test
+
+from stacklight_tests.elasticsearch_kibana import api
+
+
+@test(groups=["plugins"])
+class TestDestructiveElasticsearchPlugin(api.ElasticsearchPluginApi):
+    """Class for testing plugin failover after network disaster."""
+
+    @test(depends_on_groups=["deploy_ha_elasticsearch_kibana"],
+          groups=["check_disaster_elasticsearch_kibana",
+                  "elasticsearch_kibana", "destructive",
+                  "check_cluster_outage_elasticsearch_kibana"])
+    @log_snapshot_after_test
+    def check_cluster_outage_elasticsearch_kibana(self):
+        """Verify that the backends and dashboards recover
+        after a network outage of the whole Elasticsearch/Kibana cluster.
+
+        Scenario:
+            1. Revert the snapshot with 9 deployed nodes in HA configuration
+            2. Simulate a network outage of the whole InfluxDB/Grafana cluster
+            3. Wait for at least 7 minutes before network recovery
+            4. Wait for all services to be back online
+            5. Run OSTF
+            6. Check that the cluster's state is okay
+
+        Duration 40m
+        """
+        self.env.revert_snapshot("deploy_ha_elasticsearch_kibana")
+
+        self.helpers.emulate_whole_network_disaster(
+            delay_before_recover=7 * 60)
+
+        self.wait_plugin_online()
+
+        self.helpers.run_ostf()
+
+    @test(depends_on_groups=["deploy_elasticsearch_kibana"],
+          groups=["check_disaster_elasticsearch_kibana",
+                  "elasticsearch_kibana", "destructive",
+                  "check_node_outage_elasticsearch_kibana"])
+    @log_snapshot_after_test
+    def check_node_outage_elasticsearch_kibana(self):
+        """Verify that the backends and dashboards recover after
+        a network outage on a standalone Elasticsearch/Kibana node.
+
+        Scenario:
+            1. Revert the snapshot with 3 deployed nodes
+            2. Simulate network interruption on the Elasticsearch/Kibana node
+            3. Wait for at least 30 seconds before recover network availability
+            4. Check that plugin is working
+            5. Run OSTF
+
+        Duration 20m
+        """
+        self.env.revert_snapshot("deploy_elasticsearch_kibana")
+
+        with self.fuel_web.get_ssh_for_nailgun_node(
+                self.helpers.get_master_node_by_role(self.settings.role_name)
+        ) as remote:
+            self.remote_ops.simulate_network_interrupt_on_node(remote)
+
+        self.wait_plugin_online()
+
+        self.helpers.run_ostf()

stacklight_tests/influxdb_grafana/api.py

@@ -12,10 +12,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-from devops.helpers import helpers as devops_helpers
 from fuelweb_test import logger
 from proboscis import asserts
-import requests
 
 from stacklight_tests import base_test
 from stacklight_tests.influxdb_grafana.grafana_ui import api as ui_api
@@ -105,19 +103,6 @@ class InfluxdbPluginApi(base_test.PluginApi):
             count, nodes_count_responsed)
         asserts.assert_equal(count, nodes_count_responsed, msg)
 
-    def wait_plugin_online(self, timeout=5 * 60):
-        def check_availability():
-            try:
-                self.check_plugin_online()
-                return True
-            except (AssertionError, requests.ConnectionError):
-                return False
-
-        logger.info('Wait a plugin become online')
-        msg = "Plugin has not become online after waiting period"
-        devops_helpers.wait(
-            check_availability, timeout=timeout, timeout_msg=msg)
-
     def uninstall_plugin(self):
         return self.helpers.uninstall_plugin(self.settings.name,
                                              self.settings.version)

stacklight_tests/influxdb_grafana/test_destructive.py

@@ -47,8 +47,6 @@ class TestDestructiveInfluxdbPlugin(api.InfluxdbPluginApi):
 
         self.wait_plugin_online()
 
-        self.check_plugin_online()
-
         self.helpers.run_ostf()
 
     @test(depends_on_groups=["deploy_influxdb_grafana"],
@@ -71,11 +69,10 @@ class TestDestructiveInfluxdbPlugin(api.InfluxdbPluginApi):
         self.env.revert_snapshot("deploy_influxdb_grafana")
 
         with self.fuel_web.get_ssh_for_nailgun_node(
-                self.get_influxdb_master_node()) as remote:
+                self.helpers.get_master_node_by_role(self.settings.role_name)
+        ) as remote:
             self.remote_ops.simulate_network_interrupt_on_node(remote)
 
         self.wait_plugin_online()
 
-        self.check_plugin_online()
-
         self.helpers.run_ostf()