x/stacklight-integration-tests
Code Issues Proposed changes

Add shutdown node test for Elasticsearch/Kibana

Browse Source 
Change-Id: Idf9d245eebcb8ca35dd6bae695369cbbc3974715
This commit is contained in:
vgusev 2016-05-26 17:10:25 +03:00 committed by Vitaly Gusev
parent ca9247e091
commit 39f0fbd7c0
6 changed files with 95 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
stacklight_tests/elasticsearch_kibana/test_system.py
View File

@ -41,7 +41,6 @@ class TestNodesElasticsearchPlugin(api.ElasticsearchPluginApi):
7. Run OSTF
Duration 120m
Snapshot add_remove_controller_elasticsearch_kibana
"""
self.env.revert_snapshot("deploy_ha_elasticsearch_kibana")
@ -62,8 +61,6 @@ class TestNodesElasticsearchPlugin(api.ElasticsearchPluginApi):
self.helpers.run_ostf(should_fail=1)
self.env.make_snapshot("add_remove_controller_elasticsearch_kibana")
@test(depends_on_groups=['deploy_ha_elasticsearch_kibana'],
groups=["check_scaling_elasticsearch_kibana", "scaling",
"elasticsearch_kibana", "system",
@ -83,7 +80,6 @@ class TestNodesElasticsearchPlugin(api.ElasticsearchPluginApi):
7. Run OSTF
Duration 120m
Snapshot add_remove_compute_elasticsearch_kibana
"""
self.env.revert_snapshot("deploy_ha_elasticsearch_kibana")
@ -104,8 +100,6 @@ class TestNodesElasticsearchPlugin(api.ElasticsearchPluginApi):
self.helpers.run_ostf(should_fail=1)
self.env.make_snapshot("add_remove_compute_elasticsearch_kibana")
@test(depends_on_groups=['deploy_ha_elasticsearch_kibana'],
groups=["check_scaling_elasticsearch_kibana", "scaling",
"elasticsearch_kibana", "system",
@ -126,7 +120,6 @@ class TestNodesElasticsearchPlugin(api.ElasticsearchPluginApi):
7. Run OSTF
Duration 120m
Snapshot add_remove_elasticsearch_kibana_node
"""
self.env.revert_snapshot("deploy_ha_elasticsearch_kibana")
@ -152,7 +145,39 @@ class TestNodesElasticsearchPlugin(api.ElasticsearchPluginApi):
self.helpers.run_ostf()
self.env.make_snapshot("add_remove_elasticsearch_kibana_node")
@test(depends_on_groups=["deploy_ha_elasticsearch_kibana"],
groups=["check_failover_elasticsearch_kibana" "failover",
"elasticsearch_kibana", "system", "destructive",
"shutdown_elasticsearch_kibana_node"])
@log_snapshot_after_test
def shutdown_elasticsearch_kibana_node(self):
"""Verify that failover for Elasticsearch cluster works.
Scenario:
1. Shutdown node were es_vip_mgmt was started.
2. Check that es_vip_mgmt was started on another
elasticsearch_kibana node.
3. Check that plugin is working.
4. Check that no data lost after shutdown.
5. Run OSTF.
Duration 30m
"""
self.env.revert_snapshot("deploy_ha_elasticsearch_kibana")
vip_name = self.helpers.full_vip_name(self.settings.vip_name)
target_node = self.helpers.get_node_with_vip(
self.settings.role_name, vip_name)
self.helpers.power_off_node(target_node)
self.helpers.wait_for_vip_migration(
target_node, self.settings.role_name, vip_name)
self.check_plugin_online()
self.helpers.run_ostf()
@test(depends_on_groups=['prepare_slaves_3'],
groups=["elasticsearch_kibana_createmirror_deploy_plugin",

44
stacklight_tests/helpers/helpers.py
View File

@ -216,18 +216,46 @@ class PluginHelper(object):
if node['fqdn'] in stdout:
return node
def hard_shutdown_node(self, fqdn):
@staticmethod
def full_vip_name(vip_name):
return "".join(["vip__", vip_name])
def get_node_with_vip(self, role_name, vip, exclude_node=None):
nailgun_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
self.cluster_id, role_name)
lma_nodes = self.fuel_web.get_devops_nodes_by_nailgun_nodes(
nailgun_nodes)
lma_node = None
if exclude_node:
for node in lma_nodes:
if node.name != exclude_node.name:
lma_node = node
break
else:
lma_node = lma_nodes[0]
return self.fuel_web.get_pacemaker_resource_location(
lma_node.name, vip)[0]
def wait_for_vip_migration(self, old_master, role_name, vip,
timeout=5 * 60):
logger.info('Waiting for the migration of VIP {}'.format(vip))
msg = "VIP {0} has not been migrated away from {1}".format(
vip, old_master)
helpers.wait(
lambda: old_master != self.get_node_with_vip(
role_name, vip, exclude_node=old_master),
timeout=timeout, timeout_msg=msg)
def power_off_node(self, node):
"""Power off a node.
"""
devops_node = self.fuel_web.get_devops_node_by_nailgun_fqdn(
fqdn)
msg = 'Node {0} has not become offline after hard shutdown'.format(
devops_node.name)
logger.info('Destroy node %s', devops_node.name)
devops_node.destroy()
logger.info('Wait a %s node offline status', devops_node.name)
node.name)
logger.info('Power off node %s', node.name)
node.destroy()
logger.info('Wait a %s node offline status', node.name)
helpers.wait(lambda: not self.fuel_web.get_nailgun_node_by_devops_node(
devops_node)['online'], timeout=60 * 5, timeout_msg=msg)
node)['online'], timeout=60 * 5, timeout_msg=msg)
def emulate_whole_network_disaster(self, delay_before_recover=5 * 60,
wait_become_online=True):

13
stacklight_tests/influxdb_grafana/api.py
View File

@ -105,19 +105,6 @@ class InfluxdbPluginApi(base_test.PluginApi):
count, nodes_count_responsed)
asserts.assert_equal(count, nodes_count_responsed, msg)
def get_influxdb_master_node(self, excluded_nodes_fqdns=()):
influx_master_node = self.helpers.get_master_node_by_role(
self.settings.role_name, excluded_nodes_fqdns=excluded_nodes_fqdns)
return influx_master_node
def wait_for_rotation_influx_master(self, old_master, timeout=5 * 60):
logger.info('Wait a influxDB master node rotation')
msg = "Failed influxDB master rotation from {0}".format(old_master)
devops_helpers.wait(
lambda: old_master != self.get_influxdb_master_node(
excluded_nodes_fqdns=(old_master,))['fqdn'],
timeout=timeout, timeout_msg=msg)
def wait_plugin_online(self, timeout=5 * 60):
def check_availability():
try:

23
stacklight_tests/influxdb_grafana/test_system.py
View File

@ -164,23 +164,26 @@ class TestNodesInfluxdbPlugin(api.InfluxdbPluginApi):
"""Verify that failover for InfluxDB cluster works.
Scenario:
1. Revert snapshot with 9 deployed nodes in HA configuration
2. Determine influx_db master node were vip_influxdb was started
3. Shutdown influx_db master node
4. Check that vip_influxdb was started on another node
5. Check that plugin is working
6. Check that no data lost after shutdown
7. Run OSTF
1. Shutdown node were vip_influxdb was started.
2. Check that vip_influxdb was started on another influxdb_grafana
node.
3. Check that plugin is working.
4. Check that no data lost after shutdown.
5. Run OSTF.
Duration 30m
"""
self.env.revert_snapshot("deploy_ha_influxdb_grafana")
master_node_hostname = self.get_influxdb_master_node()['fqdn']
vip_name = self.helpers.full_vip_name(self.settings.vip_name)
self.helpers.hard_shutdown_node(master_node_hostname)
target_node = self.helpers.get_node_with_vip(
self.settings.role_name, vip_name)
self.wait_for_rotation_influx_master(master_node_hostname)
self.helpers.power_off_node(target_node)
self.helpers.wait_for_vip_migration(
target_node, self.settings.role_name, vip_name)
self.check_plugin_online()

15
stacklight_tests/lma_infrastructure_alerting/api.py
View File

@ -49,21 +49,6 @@ class InfraAlertingPluginApi(base_test.PluginApi):
self.settings.nagios_password,
self.get_plugin_vip())
def get_primary_lma_node(self, exclude=None):
nailgun_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
self.helpers.cluster_id, self.settings.role_name)
lma_nodes = self.fuel_web.get_devops_nodes_by_nailgun_nodes(
nailgun_nodes)
if exclude:
for node in lma_nodes:
if node.name != exclude:
lma_node = node
break
else:
lma_node = lma_nodes[0]
return self.fuel_web.get_pacemaker_resource_location(
lma_node.name, "vip__infrastructure_alerting_mgmt_vip")[0]
def open_nagios_page(self, link_text, anchor):
driver = self.ui_tester.get_driver(self.get_nagios_url(),
"//frame[2]", "Nagios Core")

29
stacklight_tests/lma_infrastructure_alerting/test_system.py
View File

@ -14,7 +14,6 @@
# under the License.
from fuelweb_test.helpers.decorators import log_snapshot_after_test
from proboscis import asserts
from proboscis import test
from stacklight_tests.lma_infrastructure_alerting import api
@ -143,28 +142,26 @@ class TestLMAInfraAlertingPluginSystem(api.InfraAlertingPluginApi):
"lma_infrastructure_alerting", "shutdown"])
@log_snapshot_after_test
def shutdown_infrastructure_alerting_node(self):
"""Shutdown infrastructure alerting node
"""Verify that failover for LMA Infrastructure Alerting cluster works.
Scenario:
1. Connect to any infrastructure_alerting node and run
command 'crm status'.
2. Shutdown node were vip_infrastructure_alerting_mgmt_vip
1. Shutdown node were vip_infrastructure_alerting_mgmt_vip
was started.
3. Check that vip_infrastructure_alerting was started
2. Check that vip_infrastructure_alerting was started
on another infrastructure_alerting node.
4. Check that Nagios UI works correctly.
5. Check that no data lost after shutdown.
6. Run OSTF.
3. Check that plugin is working.
4. Check that no data lost after shutdown.
5. Run OSTF.
Duration 60m
Duration 30m
"""
self.env.revert_snapshot("deploy_ha_lma_infrastructure_alerting")
target_node = self.get_primary_lma_node()
self.fuel_web.warm_shutdown_nodes([target_node])
new_node = self.get_primary_lma_node(target_node.name)
asserts.assert_not_equal(target_node, new_node)
vip_name = self.helpers.full_vip_name(self.settings.vip_name)
target_node = self.helpers.get_node_with_vip(
self.settings.role_name, vip_name)
self.helpers.power_off_node(target_node)
self.helpers.wait_for_vip_migration(
target_node, self.settings.role_name, vip_name)
self.check_plugin_online()
self.helpers.run_ostf()