From 647a66c920126367961b8d6cc038a748ae2e51df Mon Sep 17 00:00:00 2001 From: Rodion Promyshlennikov Date: Fri, 10 Jun 2016 17:21:40 +0300 Subject: [PATCH] Add test that checks work after maintenance update Change-Id: I9dcbc4bc5d4888ab23ef8d4385147bad140a55ae --- stacklight_tests/helpers/checkers.py | 4 +- stacklight_tests/helpers/helpers.py | 89 ++++++++++++++++++++++- stacklight_tests/helpers/remote_ops.py | 28 +++++++ stacklight_tests/toolchain/test_system.py | 56 +++++++++++++- 4 files changed, 169 insertions(+), 8 deletions(-) diff --git a/stacklight_tests/helpers/checkers.py b/stacklight_tests/helpers/checkers.py index 49954b9..5f8d284 100644 --- a/stacklight_tests/helpers/checkers.py +++ b/stacklight_tests/helpers/checkers.py @@ -43,8 +43,8 @@ def check_process_count(remote, process, count): :param remote: SSH connection to the node. :type remote: SSHClient - :param service_name: the process name to match. - :type service_name: str + :param process: the process name to match. + :type process: str :param count: the number of processes to match. :type count: int :returns: list of PIDs. diff --git a/stacklight_tests/helpers/helpers.py b/stacklight_tests/helpers/helpers.py index 28b0c94..61a5b61 100644 --- a/stacklight_tests/helpers/helpers.py +++ b/stacklight_tests/helpers/helpers.py @@ -21,6 +21,9 @@ from devops.helpers import helpers from fuelweb_test import logger from proboscis import asserts +from stacklight_tests.helpers import remote_ops +from stacklight_tests import settings + PLUGIN_PACKAGE_RE = re.compile(r'([^/]+)-(\d+\.\d+)-(\d+\.\d+\.\d+)') @@ -175,16 +178,16 @@ class PluginHelper(object): """Assign roles to nodes and deploy the cluster. :param nodes_roles: nodes to roles mapping. - :type name: dict + :type nodes_roles: dict :param verify_network: whether or not network verification should be run before the deployment (default: False). - :type settings: boolean + :type verify_network: boolean :param update_interfaces: whether or not interfaces should be updated before the deployment (default: True). - :type settings: boolean + :type update_interfaces: boolean :param check_services: whether or not OSTF tests should run after the deployment (default: True). - :type settings: boolean + :type check_services: boolean :returns: None """ self.fuel_web.update_nodes(self.cluster_id, nodes_roles, @@ -441,3 +444,81 @@ class PluginHelper(object): result = self.nailgun_client.put_deployment_tasks_for_cluster( self.cluster_id, data=task_ids, node_id=node_ids) self.fuel_web.assert_task_success(result, timeout=timeout) + + def apply_maintenance_update(self): + """Method applies maintenance updates on whole cluster.""" + logger.info("Applying maintenance updates on master node") + self.env.admin_install_updates() + + logger.info("Applying maintenance updates on slaves") + slaves_mu_script_url = ( + "https://github.com/Mirantis/tools-sustaining/" + "raw/master/scripts/mos_apply_mu.py") + + path_to_mu_script = "/tmp/mos_apply_mu.py" + + with self.env.d_env.get_admin_remote() as remote: + remote.check_call("wget {uri} -O {path}".format( + uri=slaves_mu_script_url, + path=path_to_mu_script) + ) + + remote.check_call( + "python {path} " + "--env-id={identifier} " + "--user={username} " + "--pass={password} " + "--tenant={tenant_name} --update".format( + path=path_to_mu_script, + identifier=self.cluster_id, + **settings.KEYSTONE_CREDS + ) + ) + + controllers = self.fuel_web.get_nailgun_cluster_nodes_by_roles( + self.cluster_id, roles=['controller', ]) + + computes = self.fuel_web.get_nailgun_cluster_nodes_by_roles( + self.cluster_id, roles=['compute', ]) + + logger.info("Restarting all OpenStack services") + + logger.info("Restarting services on controllers") + ha_services = ( + "p_heat-engine", + "p_neutron-plugin-openvswitch-agent", + "p_neutron-dhcp-agent", + "p_neutron-metadata-agent", + "p_neutron-l3-agent") + non_ha_services = ( + "heat-api-cloudwatch", + "heat-api-cfn", + "heat-api", + "cinder-api", + "cinder-scheduler", + "nova-objectstore", + "nova-cert", + "nova-api", + "nova-consoleauth", + "nova-conductor", + "nova-scheduler", + "nova-novncproxy", + "neutron-server", + ) + for controller in controllers: + with self.fuel_web.get_ssh_for_nailgun_node( + controller) as remote: + for service in ha_services: + remote_ops.manage_pacemaker_service(remote, service) + for service in non_ha_services: + remote_ops.manage_initctl_service(remote, service) + + logger.info("Restarting services on computes") + compute_services = ( + "neutron-plugin-openvswitch-agent", + "nova-compute", + ) + for compute in computes: + with self.fuel_web.get_ssh_for_nailgun_node(compute) as remote: + for service in compute_services: + remote_ops.manage_initctl_service(remote, service) diff --git a/stacklight_tests/helpers/remote_ops.py b/stacklight_tests/helpers/remote_ops.py index 05a7aaf..7f74c80 100644 --- a/stacklight_tests/helpers/remote_ops.py +++ b/stacklight_tests/helpers/remote_ops.py @@ -80,3 +80,31 @@ def get_pids_of_process(remote, name): if result['exit_code'] != 0: return [] return result['stdout'][0].strip().split() + + +def manage_pacemaker_service(remote, name, operation="restart"): + """Operate HA service on remote node. + + :param remote: SSH connection to the node. + :type remote: SSHClient + :param name: service name. + :type name: str + :param operation: type of operation, usually start, stop or restart. + :type operation: str + """ + remote.check_call("crm resource {operation} {service}".format( + operation=operation, service=name)) + + +def manage_initctl_service(remote, name, operation="restart"): + """Operate service on remote node. + + :param remote: SSH connection to the node. + :type remote: SSHClient + :param name: service name. + :type name: str + :param operation: type of operation, usually start, stop or restart. + :type operation: str + """ + remote.check_call("initctl {operation} {service}".format( + operation=operation, service=name)) diff --git a/stacklight_tests/toolchain/test_system.py b/stacklight_tests/toolchain/test_system.py index 3b3b043..3395a5c 100644 --- a/stacklight_tests/toolchain/test_system.py +++ b/stacklight_tests/toolchain/test_system.py @@ -226,7 +226,7 @@ class TestNodesToolchain(api.ToolchainApi): asserts.assert_equal( ready_nodes_hostnames_before, ready_nodes_hostnames_after, "List of ready nodes is not equal, " - "before createmirror:{}, " + "before createmirror: {}, " "after createmirror: {}.".format(ready_nodes_hostnames_before, ready_nodes_hostnames_after) ) @@ -235,13 +235,65 @@ class TestNodesToolchain(api.ToolchainApi): asserts.assert_equal( pids_after, pids_before, "PIDs of services not equal, " - "before createmirror:{}, " + "before createmirror: {}, " "after createmirror: {}.".format(pids_before, pids_after)) self.check_plugins_online() self.helpers.run_ostf() + @test(depends_on_groups=["deploy_toolchain"], + groups=["check_toolchain_after_maintenance_update", + "system", "toolchain", "maintenance_update"]) + @log_snapshot_after_test + def check_toolchain_after_maintenance_update(self): + """Check work after applying maintenance update. + + Scenario: + 1. Revert the snapshot with 3 deployed nodes + 2. Get pid of services which were launched + on controller/compute/storage/etc nodes by plugin and store them + 3. Apply maintenance update + 4. Get pid of services which were launched + on controller/compute/storage/etc nodes by plugin + and verify that they wasn't changed from last check + 5. Run OSTF + + Duration 240m + """ + self.env.revert_snapshot("deploy_toolchain") + + ready_nodes_before = self.helpers.get_all_ready_nodes() + + ready_nodes_hostnames_before = {node["hostname"] + for node in ready_nodes_before} + + pids_before = self.get_pids_of_services() + + self.helpers.apply_maintenance_update() + + ready_nodes_hostnames_after = {node["hostname"] for node + in self.helpers.get_all_ready_nodes()} + + asserts.assert_equal( + ready_nodes_hostnames_before, ready_nodes_hostnames_after, + "List of ready nodes is not equal, " + "before maintenance update: {}, " + "after maintenance update: {}.".format( + ready_nodes_hostnames_before, ready_nodes_hostnames_after) + ) + + pids_after = self.get_pids_of_services() + asserts.assert_equal( + pids_after, pids_before, + "PIDs of services not equal, " + "before maintenance update: {}, " + "after maintenance update: {}.".format(pids_before, pids_after)) + + self.check_plugins_online() + + self.helpers.run_ostf() + @test(depends_on_groups=["deploy_ha_toolchain"], groups=["shutdown_infrastructure_alerting_node_in_toolchain", "failover", "toolchain", "system", "destructive"])