Add shutdown node test for Elasticsearch/Kibana

Change-Id: Idf9d245eebcb8ca35dd6bae695369cbbc3974715

stacklight_tests/elasticsearch_kibana/test_system.py

@@ -41,7 +41,6 @@ class TestNodesElasticsearchPlugin(api.ElasticsearchPluginApi):
             7. Run OSTF
 
         Duration 120m
-        Snapshot add_remove_controller_elasticsearch_kibana
         """
         self.env.revert_snapshot("deploy_ha_elasticsearch_kibana")
 
@@ -62,8 +61,6 @@ class TestNodesElasticsearchPlugin(api.ElasticsearchPluginApi):
 
         self.helpers.run_ostf(should_fail=1)
 
-        self.env.make_snapshot("add_remove_controller_elasticsearch_kibana")
-
     @test(depends_on_groups=['deploy_ha_elasticsearch_kibana'],
           groups=["check_scaling_elasticsearch_kibana", "scaling",
                   "elasticsearch_kibana", "system",
@@ -83,7 +80,6 @@ class TestNodesElasticsearchPlugin(api.ElasticsearchPluginApi):
             7. Run OSTF
 
         Duration 120m
-        Snapshot add_remove_compute_elasticsearch_kibana
         """
         self.env.revert_snapshot("deploy_ha_elasticsearch_kibana")
 
@@ -104,8 +100,6 @@ class TestNodesElasticsearchPlugin(api.ElasticsearchPluginApi):
 
         self.helpers.run_ostf(should_fail=1)
 
-        self.env.make_snapshot("add_remove_compute_elasticsearch_kibana")
-
     @test(depends_on_groups=['deploy_ha_elasticsearch_kibana'],
           groups=["check_scaling_elasticsearch_kibana", "scaling",
                   "elasticsearch_kibana", "system",
@@ -126,7 +120,6 @@ class TestNodesElasticsearchPlugin(api.ElasticsearchPluginApi):
             7. Run OSTF
 
         Duration 120m
-        Snapshot add_remove_elasticsearch_kibana_node
         """
         self.env.revert_snapshot("deploy_ha_elasticsearch_kibana")
 
@@ -152,7 +145,39 @@ class TestNodesElasticsearchPlugin(api.ElasticsearchPluginApi):
 
         self.helpers.run_ostf()
 
-        self.env.make_snapshot("add_remove_elasticsearch_kibana_node")
+    @test(depends_on_groups=["deploy_ha_elasticsearch_kibana"],
+          groups=["check_failover_elasticsearch_kibana" "failover",
+                  "elasticsearch_kibana", "system", "destructive",
+                  "shutdown_elasticsearch_kibana_node"])
+    @log_snapshot_after_test
+    def shutdown_elasticsearch_kibana_node(self):
+        """Verify that failover for Elasticsearch cluster works.
+
+        Scenario:
+            1. Shutdown node were es_vip_mgmt was started.
+            2. Check that es_vip_mgmt was started on another
+               elasticsearch_kibana node.
+            3. Check that plugin is working.
+            4. Check that no data lost after shutdown.
+            5. Run OSTF.
+
+        Duration 30m
+        """
+        self.env.revert_snapshot("deploy_ha_elasticsearch_kibana")
+
+        vip_name = self.helpers.full_vip_name(self.settings.vip_name)
+
+        target_node = self.helpers.get_node_with_vip(
+            self.settings.role_name, vip_name)
+
+        self.helpers.power_off_node(target_node)
+
+        self.helpers.wait_for_vip_migration(
+            target_node, self.settings.role_name, vip_name)
+
+        self.check_plugin_online()
+
+        self.helpers.run_ostf()
 
     @test(depends_on_groups=['prepare_slaves_3'],
           groups=["elasticsearch_kibana_createmirror_deploy_plugin",

stacklight_tests/helpers/helpers.py

@@ -216,18 +216,46 @@ class PluginHelper(object):
             if node['fqdn'] in stdout:
                 return node
 
-    def hard_shutdown_node(self, fqdn):
+    @staticmethod
+    def full_vip_name(vip_name):
+        return "".join(["vip__", vip_name])
+
+    def get_node_with_vip(self, role_name, vip, exclude_node=None):
+        nailgun_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
+            self.cluster_id, role_name)
+        lma_nodes = self.fuel_web.get_devops_nodes_by_nailgun_nodes(
+            nailgun_nodes)
+        lma_node = None
+        if exclude_node:
+            for node in lma_nodes:
+                if node.name != exclude_node.name:
+                    lma_node = node
+                    break
+        else:
+            lma_node = lma_nodes[0]
+        return self.fuel_web.get_pacemaker_resource_location(
+            lma_node.name, vip)[0]
+
+    def wait_for_vip_migration(self, old_master, role_name, vip,
+                               timeout=5 * 60):
+        logger.info('Waiting for the migration of VIP {}'.format(vip))
+        msg = "VIP {0} has not been migrated away from {1}".format(
+            vip, old_master)
+        helpers.wait(
+            lambda: old_master != self.get_node_with_vip(
+                role_name, vip, exclude_node=old_master),
+            timeout=timeout, timeout_msg=msg)
+
+    def power_off_node(self, node):
         """Power off a node.
         """
-        devops_node = self.fuel_web.get_devops_node_by_nailgun_fqdn(
-            fqdn)
         msg = 'Node {0} has not become offline after hard shutdown'.format(
-            devops_node.name)
+            node.name)
-        logger.info('Destroy node %s', devops_node.name)
+        logger.info('Power off node %s', node.name)
-        devops_node.destroy()
+        node.destroy()
-        logger.info('Wait a %s node offline status', devops_node.name)
+        logger.info('Wait a %s node offline status', node.name)
         helpers.wait(lambda: not self.fuel_web.get_nailgun_node_by_devops_node(
-            devops_node)['online'], timeout=60 * 5, timeout_msg=msg)
+            node)['online'], timeout=60 * 5, timeout_msg=msg)
 
     def emulate_whole_network_disaster(self, delay_before_recover=5 * 60,
                                        wait_become_online=True):

stacklight_tests/influxdb_grafana/api.py

@@ -105,19 +105,6 @@ class InfluxdbPluginApi(base_test.PluginApi):
             count, nodes_count_responsed)
         asserts.assert_equal(count, nodes_count_responsed, msg)
 
-    def get_influxdb_master_node(self, excluded_nodes_fqdns=()):
-        influx_master_node = self.helpers.get_master_node_by_role(
-            self.settings.role_name, excluded_nodes_fqdns=excluded_nodes_fqdns)
-        return influx_master_node
-
-    def wait_for_rotation_influx_master(self, old_master, timeout=5 * 60):
-        logger.info('Wait a influxDB master node rotation')
-        msg = "Failed influxDB master rotation from {0}".format(old_master)
-        devops_helpers.wait(
-            lambda: old_master != self.get_influxdb_master_node(
-                excluded_nodes_fqdns=(old_master,))['fqdn'],
-            timeout=timeout, timeout_msg=msg)
-
     def wait_plugin_online(self, timeout=5 * 60):
         def check_availability():
             try:

stacklight_tests/influxdb_grafana/test_system.py

@@ -164,23 +164,26 @@ class TestNodesInfluxdbPlugin(api.InfluxdbPluginApi):
         """Verify that failover for InfluxDB cluster works.
 
         Scenario:
-            1. Revert snapshot with 9 deployed nodes in HA configuration
+            1. Shutdown node were vip_influxdb was started.
-            2. Determine influx_db master node were vip_influxdb was started
+            2. Check that vip_influxdb was started on another influxdb_grafana
-            3. Shutdown influx_db master node
+               node.
-            4. Check that vip_influxdb was started on another node
+            3. Check that plugin is working.
-            5. Check that plugin is working
+            4. Check that no data lost after shutdown.
-            6. Check that no data lost after shutdown
+            5. Run OSTF.
-            7. Run OSTF
 
         Duration 30m
         """
         self.env.revert_snapshot("deploy_ha_influxdb_grafana")
 
-        master_node_hostname = self.get_influxdb_master_node()['fqdn']
+        vip_name = self.helpers.full_vip_name(self.settings.vip_name)
 
-        self.helpers.hard_shutdown_node(master_node_hostname)
+        target_node = self.helpers.get_node_with_vip(
+            self.settings.role_name, vip_name)
 
-        self.wait_for_rotation_influx_master(master_node_hostname)
+        self.helpers.power_off_node(target_node)
+
+        self.helpers.wait_for_vip_migration(
+            target_node, self.settings.role_name, vip_name)
 
         self.check_plugin_online()
 

stacklight_tests/lma_infrastructure_alerting/api.py

@@ -49,21 +49,6 @@ class InfraAlertingPluginApi(base_test.PluginApi):
                                                 self.settings.nagios_password,
                                                 self.get_plugin_vip())
 
-    def get_primary_lma_node(self, exclude=None):
-        nailgun_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
-            self.helpers.cluster_id, self.settings.role_name)
-        lma_nodes = self.fuel_web.get_devops_nodes_by_nailgun_nodes(
-            nailgun_nodes)
-        if exclude:
-            for node in lma_nodes:
-                if node.name != exclude:
-                    lma_node = node
-                    break
-        else:
-            lma_node = lma_nodes[0]
-        return self.fuel_web.get_pacemaker_resource_location(
-            lma_node.name, "vip__infrastructure_alerting_mgmt_vip")[0]
-
     def open_nagios_page(self, link_text, anchor):
         driver = self.ui_tester.get_driver(self.get_nagios_url(),
                                            "//frame[2]", "Nagios Core")

stacklight_tests/lma_infrastructure_alerting/test_system.py

@@ -14,7 +14,6 @@
 #    under the License.
 
 from fuelweb_test.helpers.decorators import log_snapshot_after_test
-from proboscis import asserts
 from proboscis import test
 
 from stacklight_tests.lma_infrastructure_alerting import api
@@ -143,28 +142,26 @@ class TestLMAInfraAlertingPluginSystem(api.InfraAlertingPluginApi):
                   "lma_infrastructure_alerting", "shutdown"])
     @log_snapshot_after_test
     def shutdown_infrastructure_alerting_node(self):
-        """Shutdown infrastructure alerting node
+        """Verify that failover for LMA Infrastructure Alerting cluster works.
 
         Scenario:
-            1. Connect to any infrastructure_alerting node and run
+            1. Shutdown node were vip_infrastructure_alerting_mgmt_vip
-               command 'crm status'.
-            2. Shutdown node were vip_infrastructure_alerting_mgmt_vip
                was started.
-            3. Check that vip_infrastructure_alerting was started
+            2. Check that vip_infrastructure_alerting was started
                on another infrastructure_alerting node.
-            4. Check that Nagios UI works correctly.
+            3. Check that plugin is working.
-            5. Check that no data lost after shutdown.
+            4. Check that no data lost after shutdown.
-            6. Run OSTF.
+            5. Run OSTF.
 
-        Duration 60m
+        Duration 30m
         """
         self.env.revert_snapshot("deploy_ha_lma_infrastructure_alerting")
-
+        vip_name = self.helpers.full_vip_name(self.settings.vip_name)
-        target_node = self.get_primary_lma_node()
+        target_node = self.helpers.get_node_with_vip(
-        self.fuel_web.warm_shutdown_nodes([target_node])
+            self.settings.role_name, vip_name)
-        new_node = self.get_primary_lma_node(target_node.name)
+        self.helpers.power_off_node(target_node)
-        asserts.assert_not_equal(target_node, new_node)
+        self.helpers.wait_for_vip_migration(
-
+            target_node, self.settings.role_name, vip_name)
         self.check_plugin_online()
         self.helpers.run_ostf()