From d4b943b4ff3b7def08ffffe43bcc918d86578c89 Mon Sep 17 00:00:00 2001 From: Igor Degtiarov Date: Tue, 27 Sep 2016 15:10:17 +0300 Subject: [PATCH] Add autoscaling test for Telemetry plugin Change-Id: I3d806358ba1a557d8b2765a808a0e6585f4fe74c --- .../heat_autoscaling_neutron.yaml | 75 +++++ .../heat_autoscaling_nova.yaml | 71 +++++ stacklight_tests/openstack_telemetry/api.py | 289 +++++++++++++++++- .../toolchain/test_openstack_telemetry.py | 1 + 4 files changed, 422 insertions(+), 14 deletions(-) create mode 100644 fixtures/autoscaling_templates/heat_autoscaling_neutron.yaml create mode 100644 fixtures/autoscaling_templates/heat_autoscaling_nova.yaml diff --git a/fixtures/autoscaling_templates/heat_autoscaling_neutron.yaml b/fixtures/autoscaling_templates/heat_autoscaling_neutron.yaml new file mode 100644 index 0000000..1ccaea9 --- /dev/null +++ b/fixtures/autoscaling_templates/heat_autoscaling_neutron.yaml @@ -0,0 +1,75 @@ +heat_template_version: 2013-05-23 + +parameters: + KeyName: + type: string + InstanceType: + type: string + ImageId: + type: string + SecurityGroup: + type: string + Net: + type: string + +resources: + my_asg: + type: OS::Heat::AutoScalingGroup + properties: + resource: + type: OS::Nova::Server + properties: + metadata: {"metering.stack": {get_param: "OS::stack_id"}} + key_name: { get_param: KeyName } + image: { get_param: ImageId } + flavor: { get_param: InstanceType } + security_groups: + - get_param: SecurityGroup + networks: + - network: {get_param: Net} + min_size: 1 + max_size: 3 + + scale_up_policy: + type: OS::Heat::ScalingPolicy + properties: + adjustment_type: change_in_capacity + auto_scaling_group_id: {get_resource: my_asg} + cooldown: 60 + scaling_adjustment: 2 + + scale_down_policy: + type: OS::Heat::ScalingPolicy + properties: + adjustment_type: change_in_capacity + auto_scaling_group_id: {get_resource: my_asg} + cooldown: 60 + scaling_adjustment: '-1' + + cpu_alarm_high: + type: OS::Ceilometer::Alarm + properties: + description: Scale-up if count of instance <= 1 for 1 minute + meter_name: network.incoming.bytes + statistic: count + period: 60 + evaluation_periods: 1 + threshold: 1 + alarm_actions: + - {get_attr: [scale_up_policy, alarm_url]} + matching_metadata: {'metadata.user_metadata.stack': {get_param: "OS::stack_id"}} + comparison_operator: le + + cpu_alarm_low: + type: OS::Ceilometer::Alarm + properties: + description: Scale-down if maximum count of instance > 2 for 1 minutes + meter_name: network.incoming.bytes + statistic: count + period: 60 + evaluation_periods: 1 + threshold: 2 + alarm_actions: + - {get_attr: [scale_down_policy, alarm_url]} + matching_metadata: {'metadata.user_metadata.stack': {get_param: "OS::stack_id"}} + comparison_operator: gt diff --git a/fixtures/autoscaling_templates/heat_autoscaling_nova.yaml b/fixtures/autoscaling_templates/heat_autoscaling_nova.yaml new file mode 100644 index 0000000..bd55bbc --- /dev/null +++ b/fixtures/autoscaling_templates/heat_autoscaling_nova.yaml @@ -0,0 +1,71 @@ +heat_template_version: 2013-05-23 + +parameters: + KeyName: + type: string + InstanceType: + type: string + ImageId: + type: string + SecurityGroup: + type: string + +resources: + my_asg: + type: OS::Heat::AutoScalingGroup + properties: + resource: + type: OS::Nova::Server + properties: + metadata: {"metering.stack": {get_param: "OS::stack_id"}} + key_name: { get_param: KeyName } + image: { get_param: ImageId } + flavor: { get_param: InstanceType } + security_groups: + - get_param: SecurityGroup + min_size: 1 + max_size: 3 + + scale_up_policy: + type: OS::Heat::ScalingPolicy + properties: + adjustment_type: change_in_capacity + auto_scaling_group_id: {get_resource: my_asg} + cooldown: 60 + scaling_adjustment: 2 + + scale_down_policy: + type: OS::Heat::ScalingPolicy + properties: + adjustment_type: change_in_capacity + auto_scaling_group_id: {get_resource: my_asg} + cooldown: 60 + scaling_adjustment: '-1' + + cpu_alarm_high: + type: OS::Ceilometer::Alarm + properties: + description: Scale-up if count of instance <= 1 for 1 minute + meter_name: network.incoming.bytes + statistic: count + period: 60 + evaluation_periods: 1 + threshold: 1 + alarm_actions: + - {get_attr: [scale_up_policy, alarm_url]} + matching_metadata: {'metadata.user_metadata.stack': {get_param: "OS::stack_id"}} + comparison_operator: le + + cpu_alarm_low: + type: OS::Ceilometer::Alarm + properties: + description: Scale-down if maximum count of instance > 2 for 1 minutes + meter_name: network.incoming.bytes + statistic: count + period: 60 + evaluation_periods: 1 + threshold: 2 + alarm_actions: + - {get_attr: [scale_down_policy, alarm_url]} + matching_metadata: {'metadata.user_metadata.stack': {get_param: "OS::stack_id"}} + comparison_operator: gt diff --git a/stacklight_tests/openstack_telemetry/api.py b/stacklight_tests/openstack_telemetry/api.py index 4d00580..6863b33 100644 --- a/stacklight_tests/openstack_telemetry/api.py +++ b/stacklight_tests/openstack_telemetry/api.py @@ -12,15 +12,18 @@ # License for the specific language governing permissions and limitations # under the License. import datetime +import os import ceilometerclient.v2.client from fuelweb_test.helpers import checkers as fuelweb_checkers from fuelweb_test import logger +import heatclient.v1.client +import proboscis from stacklight_tests import base_test from stacklight_tests.helpers import checkers from stacklight_tests.helpers import helpers -from stacklight_tests.influxdb_grafana.api import InfluxdbPluginApi +from stacklight_tests.influxdb_grafana import api as influx_api from stacklight_tests.openstack_telemetry import plugin_settings @@ -28,35 +31,62 @@ class OpenstackTelemeteryPluginApi(base_test.PluginApi): def __init__(self): super(OpenstackTelemeteryPluginApi, self).__init__() self._ceilometer = None + self._heat_cli = None + + @property + def keystone_access(self): + return self.helpers.os_conn.keystone_access + + @property + def nova_cli(self): + return self.helpers.os_conn.nova + + @property + def neutron_cli(self): + return self.helpers.os_conn.neutron + + @property + def auth_url(self): + return self.keystone_access.service_catalog.url_for( + service_type='identity', service_name='keystone', + interface='internal') + + @property + def heat_cli(self): + if self._heat_cli is None: + endpoint = self.keystone_access.service_catalog.url_for( + service_type='orchestration', service_name='heat', + interface='internal') + if not endpoint: + raise self.helpers.NotFound( + "Cannot find Heat endpoint") + self._heat_cli = heatclient.v1.client.Client( + auth_url=self.auth_url, + endpoint=endpoint, + token=(lambda: self.keystone_access.auth_token)() + ) + return self._heat_cli @property def ceilometer_client(self): if self._ceilometer is None: - keystone_access = self.helpers.os_conn.keystone_access - endpoint = keystone_access.service_catalog.url_for( + endpoint = self.keystone_access.service_catalog.url_for( service_type='metering', service_name='ceilometer', interface='internal') if not endpoint: raise self.helpers.NotFound( "Cannot find Ceilometer endpoint") - aodh_endpoint = keystone_access.service_catalog.url_for( + aodh_endpoint = self.keystone_access.service_catalog.url_for( service_type='alarming', service_name='aodh', interface='internal') if not aodh_endpoint: raise self.helpers.NotFound( "Cannot find AODH (alarm) endpoint") - auth_url = keystone_access.service_catalog.url_for( - service_type='identity', service_name='keystone', - interface='internal') - if not auth_url: - raise self.helpers.NotFound( - "Cannot find Keystone endpoint") - self._ceilometer = ceilometerclient.v2.Client( aodh_endpoint=aodh_endpoint, - auth_url=auth_url, + auth_url=self.auth_url, endpoint=endpoint, - token=lambda: keystone_access.auth_token) + token=lambda: self.keystone_access.auth_token) return self._ceilometer def get_plugin_settings(self): @@ -126,7 +156,7 @@ class OpenstackTelemeteryPluginApi(base_test.PluginApi): checkers.check_http_get_response("{}/v2/capabilities".format(endpoint), headers=headers) logger.info("Check Ceilometer database in InfluxDB") - InfluxdbPluginApi().do_influxdb_query( + influx_api.InfluxdbPluginApi().do_influxdb_query( "show measurements", db="ceilometer") def uninstall_plugin(self): @@ -268,6 +298,172 @@ class OpenstackTelemeteryPluginApi(base_test.PluginApi): self.helpers.verify(60, self.ceilometer_client.meters.list, 4, fail_msg, msg, limit=10, unique=True) + def check_ceilometer_autoscaling(self): + logger.info("Start checking autoscaling") + + # check required resources available + self._check_required_resources() + + # create test flavor + fail_msg = "Failed to create test heat flavor" + heat_flavor = self.helpers.verify( + 60, self.helpers.os_conn.create_flavor, 1, fail_msg, + "creating test heat flavor", + name="ostf_test-flavor-autoscaling", ram=256, vcpus=1, disk=2 + ) + + # create keypair + fail_msg = "Failed to create test keypair" + keypair = self.helpers.verify( + 60, self.helpers.os_conn.create_key, 2, fail_msg, + "creating test keypair", key_name="ostf_test-keypair-autoscaling") + + # create security group + fail_msg = "Failed to create test seurity group" + msg = "creating test security group" + sec_group = self.helpers.verify(60, self._create_securtity_group, 3, + fail_msg, msg) + parameters = { + 'KeyName': keypair.name, + 'InstanceType': heat_flavor.name, + 'ImageId': "TestVM", + 'SecurityGroup': sec_group.name + } + net_provider = self.helpers.nailgun_client.get_cluster( + self.helpers.cluster_id)["net_provider"] + + if "neutron" in net_provider: + template = self._load_template("heat_autoscaling_neutron.yaml") + fail_msg = "Failed to create test network resources" + msg = "creating network resources" + parameters['Net'] = self.helpers.verify( + 60, self._create_network_resources, 4, fail_msg, msg, + tenant_id=self.keystone_access.tenant_id) + else: + template = self._load_temlate("heat_autoscaling_nova.yaml") + + # create Heat stack + fail_msg = "Failed to create Heat stack" + msg = "creating Heat stack" + stack_name = 'ostf_test-heat-stack' + stack_id = self.helpers.verify(60, self.heat_cli.stacks.create, 5, + fail_msg, msg, + stack_name=stack_name, + template=template, + parameters=parameters, + disable_rollback=True)['stack']['id'] + + # get Heat stack + fail_msg = "Failed to get Heat stack" + msg = "getting Heat stack" + stack = self.helpers.verify(60, self.heat_cli.stacks.get, 6, fail_msg, + msg, stack_id=stack_id) + + # check stack creation comleted + fail_msg = "Stack was not created properly." + self.helpers.verify( + 600, self._check_stack_status, + 6, fail_msg, + "stack status becoming 'CREATE_COMPLETE'", + stack_id=stack_id, status="CREATE_COMPLETE" + ) + + # getting instances list + reduced_stack_name = "{0}-{1}".format( + stack.stack_name[:2], stack.stack_name[-4:]) + + instances = self.helpers.verify(60, self._get_instances_by_name_mask, + 7, "Failed to get instances list", + "getting instances list", + mask_name=reduced_stack_name) + + # launching the second instance during autoscaling + fail_msg = "Failed to launch the 2nd instance per autoscaling alarm." + msg = "launching the new instance per autoscaling alarm" + self.helpers.verify( + 1500, self._check_instance_scaling, 8, fail_msg, msg, + exp_lenght=(len(instances) + 2), + reduced_stack_name=reduced_stack_name + ) + + # termination of the second instance during autoscaling + fail_msg = ("Failed to terminate the 2nd instance per autoscaling " + "alarm.") + msg = "terminating the 2nd instance per autoscaling alarm" + self.helpers.verify( + 1500, self._check_instance_scaling, 9, fail_msg, msg, + exp_lenght=(len(instances) + 1), + reduced_stack_name=reduced_stack_name + ) + + # delete Heat stack + self.helpers.verify(60, self.heat_cli.stacks.delete, 10, + "Failed to delete Heat stack", + "deleting Heat stack", stack_id=stack_id) + self.helpers.verify( + 600, self._check_instance_scaling, 11, + "Not all stack instances was deleted", + "checking all stack instances was deleted", + exp_lenght=(len(instances) - 1), + reduced_stack_name=reduced_stack_name + ) + + def _create_securtity_group(self, name="ostf_test-secgroup-autoscaling"): + logger.info("Creating test security group for Heat autoscaling...") + sg_desc = name + " description" + sec_group = None + for sgp in self.nova_cli.security_groups.list(): + if name == sgp.name: + sec_group = sgp + break + if not sec_group: + sec_group = self.nova_cli.security_groups.create(name, sg_desc) + return sec_group + + def _create_network_resources(self, tenant_id): + """This method creates network resources. + + It creates a network, an internal subnet on the network, a router and + links the network to the router. All resources created by this method + will be automatically deleted. + """ + logger.info("Creating network resources...") + net_name = "ostf-autoscaling-test-service-net" + net_body = { + "network": { + "name": net_name, + "tenant_id": tenant_id + } + } + ext_net = None + net = None + for network in self.neutron_cli.list_networks()["networks"]: + if not net and network["name"] == net_name: + net = network + if not ext_net and network["router:external"]: + ext_net = network + if not net: + net = self.neutron_cli.create_network(net_body)["network"] + subnet = self.helpers.os_conn.create_subnet( + "sub" + net_name, net["id"], "10.1.7.0/24", tenant_id=tenant_id + ) + router_name = 'ostf-autoscaling-test-service-router' + router = self.helpers.os_conn.create_router( + router_name, self.helpers.os_conn.get_tenant("admin")) + self.neutron_cli.add_interface_router( + router["id"], {"subnet_id": subnet["id"]}) + return net["id"] + + @staticmethod + def _load_template(file_name): + """Load specified template file from etc directory.""" + + filepath = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + '../../fixtures/autoscaling_templates', file_name) + with open(filepath) as f: + return f.read() + def create_alarm(self, **kwargs): for alarm in self.ceilometer_client.alarms.list(): if alarm.name == kwargs['name']: @@ -286,3 +482,68 @@ class OpenstackTelemeteryPluginApi(base_test.PluginApi): elif alarm_state == 'alarm' or 'ok': return True return False + + def _check_instance_scaling(self, exp_lenght, reduced_stack_name): + return exp_lenght == len(self._get_instances_by_name_mask( + reduced_stack_name)) + + def _check_stack_status(self, stack_id, status): + try: + stack_status = self.heat_cli.stacks.get(stack_id).stack_status + except Exception: + stack_status = None + if stack_status and stack_status == status: + return True + return False + + def _get_instances_by_name_mask(self, mask_name): + """This method retuns list of instances with certain names.""" + + instances = [] + + instance_list = self.nova_cli.servers.list() + logger.info('Instances list is {0}'.format(instance_list)) + logger.info( + 'Expected instance name should inlude {0}'.format(mask_name)) + + for inst in instance_list: + logger.info('Instance name is {0}'.format(inst.name)) + if inst.name.startswith(mask_name): + instances.append(inst) + return instances + + def _get_info_about_available_resources(self, min_ram, min_hdd, min_vcpus): + """This function allows to get the information about resources. + + We need to collect the information about available RAM, HDD and vCPUs + on all compute nodes for cases when we will create more than 1 VM. + + This function returns the count of VMs with required parameters which + we can successfully run on existing cloud. + """ + vms_count = 0 + for hypervisor in self.nova_cli.hypervisors.list(): + if hypervisor.free_ram_mb >= min_ram: + if hypervisor.free_disk_gb >= min_hdd: + if hypervisor.vcpus - hypervisor.vcpus_used >= min_vcpus: + # We need to determine how many VMs we can run + # on this hypervisor + free_cpu = hypervisor.vcpus - hypervisor.vcpus_used + k1 = int(hypervisor.free_ram_mb / min_ram) + k2 = int(hypervisor.free_disk_gb / min_hdd) + k3 = int(free_cpu / min_vcpus) + vms_count += min(k1, k2, k3) + return vms_count + + def _check_required_resources(self, min_required_ram_mb=4096, + hdd=40, vCpu=2): + vms_count = self._get_info_about_available_resources( + min_required_ram_mb, hdd, vCpu) + if vms_count < 1: + msg = ('This test requires more hardware resources of your ' + 'OpenStack cluster: your cloud should allow to create ' + 'at least 1 VM with {0} MB of RAM, {1} HDD and {2} vCPUs. ' + 'You need to remove some resources or add compute nodes ' + 'to have an ability to run this OSTF test.' + .format(min_required_ram_mb, hdd, vCpu)) + raise proboscis.SkipTest(msg) diff --git a/stacklight_tests/toolchain/test_openstack_telemetry.py b/stacklight_tests/toolchain/test_openstack_telemetry.py index 36c01f3..ffd7e98 100644 --- a/stacklight_tests/toolchain/test_openstack_telemetry.py +++ b/stacklight_tests/toolchain/test_openstack_telemetry.py @@ -74,6 +74,7 @@ class TestOpenstackTelemetry(api.ToolchainApi): self.helpers.run_ostf(test_sets=test_sets) self.OPENSTACK_TELEMETRY.check_ceilometer_sample_functionality() self.OPENSTACK_TELEMETRY.check_ceilometer_alarm_functionality() + self.OPENSTACK_TELEMETRY.check_ceilometer_autoscaling() if additional_tests: for ostf_test in additional_tests: ostf_test()