From 65b3c5fff2ba6192a97d34f7080ea3c32b9ff31b Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 16 Jun 2015 09:07:56 -0500 Subject: [PATCH] Add Heat scenario to test scaling policies Scenario triggers scaling policy webhooks and tracks their performance. Change-Id: I82270c1ceb2bf99254581b0b57f152a1361298d2 --- etc/rally/rally.conf.sample | 7 ++ .../extra/autoscaling_group.yaml.template | 46 ++++++++++++ rally-jobs/heat.yaml | 38 ++++++++++ .../openstack/scenarios/heat/stacks.py | 37 ++++++++++ .../plugins/openstack/scenarios/heat/utils.py | 72 ++++++++++++++++++- .../heat/create-stack-and-scale.json | 22 ++++++ .../heat/create-stack-and-scale.yaml | 15 ++++ .../openstack/scenarios/heat/test_stacks.py | 19 +++++ .../openstack/scenarios/heat/test_utils.py | 54 ++++++++++++++ 9 files changed, 309 insertions(+), 1 deletion(-) create mode 100644 rally-jobs/extra/autoscaling_group.yaml.template create mode 100644 samples/tasks/scenarios/heat/create-stack-and-scale.json create mode 100644 samples/tasks/scenarios/heat/create-stack-and-scale.yaml diff --git a/etc/rally/rally.conf.sample b/etc/rally/rally.conf.sample index 7a8742f8..b072ab17 100644 --- a/etc/rally/rally.conf.sample +++ b/etc/rally/rally.conf.sample @@ -220,6 +220,13 @@ # Time interval(in sec) between checks when waiting for stack to be restored. #heat_stack_restore_poll_interval = 1.0 +# Time (in sec) to wait for stack to scale up or down. (floating point value) +#heat_stack_scale_timeout = 3600.0 + +# Time interval (in sec) between checks when waiting for a stack to +# scale up or down. (floating point value) +#heat_stack_scale_poll_interval = 1.0 + # Delay between creating Manila share and polling for its status. # (floating point value) #manila_share_create_prepoll_delay = 2.0 diff --git a/rally-jobs/extra/autoscaling_group.yaml.template b/rally-jobs/extra/autoscaling_group.yaml.template new file mode 100644 index 00000000..f6f9f124 --- /dev/null +++ b/rally-jobs/extra/autoscaling_group.yaml.template @@ -0,0 +1,46 @@ +heat_template_version: 2013-05-23 + +parameters: + flavor: + type: string + default: m1.tiny + constraints: + - custom_constraint: nova.flavor + image: + type: string + default: cirros-0.3.4-x86_64-uec + constraints: + - custom_constraint: glance.image + scaling_adjustment: + type: number + default: 1 + max_size: + type: number + default: 5 + constraints: + - range: {min: 1} + + +resources: + asg: + type: OS::Heat::AutoScalingGroup + properties: + resource: + type: OS::Nova::Server + properties: + image: { get_param: image } + flavor: { get_param: flavor } + min_size: 1 + desired_capacity: 3 + max_size: { get_param: max_size } + + scaling_policy: + type: OS::Heat::ScalingPolicy + properties: + adjustment_type: change_in_capacity + auto_scaling_group_id: {get_resource: asg} + scaling_adjustment: { get_param: scaling_adjustment } + +outputs: + scaling_url: + value: {get_attr: [scaling_policy, alarm_url]} diff --git a/rally-jobs/heat.yaml b/rally-jobs/heat.yaml index 74d707a1..4327ef48 100644 --- a/rally-jobs/heat.yaml +++ b/rally-jobs/heat.yaml @@ -242,3 +242,41 @@ sla: failure_rate: max: 0 + + HeatStacks.create_stack_and_scale: + - + args: + template_path: "/home/jenkins/.rally/extra/autoscaling_group.yaml.template" + output_key: "scaling_url" + delta: 1 + parameters: + scaling_adjustment: 1 + runner: + type: "constant" + times: 2 + concurrency: 1 + context: + users: + tenants: 2 + users_per_tenant: 1 + sla: + failure_rate: + max: 0 + - + args: + template_path: "/home/jenkins/.rally/extra/autoscaling_group.yaml.template" + output_key: "scaling_url" + delta: -1 + parameters: + scaling_adjustment: -1 + runner: + type: "constant" + times: 2 + concurrency: 2 + context: + users: + tenants: 2 + users_per_tenant: 1 + sla: + failure_rate: + max: 0 diff --git a/rally/plugins/openstack/scenarios/heat/stacks.py b/rally/plugins/openstack/scenarios/heat/stacks.py index 45340669..f5399fd7 100644 --- a/rally/plugins/openstack/scenarios/heat/stacks.py +++ b/rally/plugins/openstack/scenarios/heat/stacks.py @@ -140,6 +140,43 @@ class HeatStacks(utils.HeatScenario): updated_environment or environment) self._delete_stack(stack) + @types.set(template_path=types.FileType, files=types.FileTypeDict) + @validation.required_services(consts.Service.HEAT) + @validation.required_openstack(users=True) + @base.scenario(context={"cleanup": ["heat"]}) + def create_stack_and_scale(self, template_path, output_key, delta, + parameters=None, files=None, + environment=None): + """Create an autoscaling stack and invoke a scaling policy. + + Measure the performance of autoscaling webhooks. + + :param template_path: path to template file that includes an + OS::Heat::AutoScalingGroup resource + :param output_key: the stack output key that corresponds to + the scaling webhook + :param delta: the number of instances the stack is expected to + change by. + :param parameters: parameters to use in heat template + :param files: files used in template (dict of file name to + file path) + :param environment: stack environment definition (dict) + """ + # TODO(stpierre): Kilo Heat is *much* better than Juno for the + # requirements of this scenario, so once Juno goes out of + # support we should update this scenario to suck less. Namely: + # + # * Kilo Heat can supply alarm_url attributes without needing + # an output key, so instead of getting the output key from + # the user, just get the name of the ScalingPolicy to apply. + # * Kilo Heat changes the status of a stack while scaling it, + # so _scale_stack() can check for the stack to have changed + # size and for it to be in UPDATE_COMPLETE state, so the + # user no longer needs to specify the expected delta. + stack = self._create_stack(template_path, parameters, files, + environment) + self._scale_stack(stack, output_key, delta) + @types.set(template_path=types.FileType, files=types.FileTypeDict) @validation.required_services(consts.Service.HEAT) @validation.required_openstack(users=True) diff --git a/rally/plugins/openstack/scenarios/heat/utils.py b/rally/plugins/openstack/scenarios/heat/utils.py index e0254011..eee59ea5 100644 --- a/rally/plugins/openstack/scenarios/heat/utils.py +++ b/rally/plugins/openstack/scenarios/heat/utils.py @@ -16,10 +16,14 @@ import time from oslo_config import cfg +import requests +from rally.common import log as logging +from rally import exceptions from rally.task.scenarios import base from rally.task import utils +LOG = logging.getLogger(__name__) HEAT_BENCHMARK_OPTS = [ cfg.FloatOpt("heat_stack_create_prepoll_delay", @@ -87,7 +91,14 @@ HEAT_BENCHMARK_OPTS = [ cfg.FloatOpt("heat_stack_restore_poll_interval", default=1.0, help="Time interval(in sec) between checks when waiting for " - "stack to be restored.") + "stack to be restored."), + cfg.FloatOpt("heat_stack_scale_timeout", + default=3600.0, + help="Time (in sec) to wait for stack to scale up or down."), + cfg.FloatOpt("heat_stack_scale_poll_interval", + default=1.0, + help="Time interval (in sec) between checks when waiting for " + "a stack to scale up or down."), ] CONF = cfg.CONF @@ -272,3 +283,62 @@ class HeatScenario(base.Scenario): timeout=CONF.benchmark.heat_stack_restore_timeout, check_interval=CONF.benchmark.heat_stack_restore_poll_interval ) + + def _count_instances(self, stack): + """Count instances in a Heat stack. + + :param stack: stack to count instances in. + """ + return len([ + r for r in self.clients("heat").resources.list(stack.id, + nested_depth=1) + if r.resource_type == "OS::Nova::Server"]) + + def _scale_stack(self, stack, output_key, delta): + """Scale a stack up or down. + + Calls the webhook given in the output value identified by + 'output_key', and waits for the stack size to change by + 'delta'. + + :param stack: stack to scale up or down + :param output_key: The name of the output to get the URL from + :param delta: The expected change in number of instances in + the stack (signed int) + """ + num_instances = self._count_instances(stack) + expected_instances = num_instances + delta + LOG.debug("Scaling stack %s from %s to %s instances with %s" % + (stack.id, num_instances, expected_instances, output_key)) + with base.AtomicAction(self, "heat.scale_with_%s" % output_key): + self._stack_webhook(stack, output_key) + utils.wait_for( + stack, + is_ready=lambda s: ( + self._count_instances(s) == expected_instances), + update_resource=utils.get_from_manager( + ["UPDATE_FAILED"]), + timeout=CONF.benchmark.heat_stack_scale_timeout, + check_interval=CONF.benchmark.heat_stack_scale_poll_interval) + + def _stack_webhook(self, stack, output_key): + """POST to the URL given in the output value identified by output_key. + + This can be used to scale stacks up and down, for instance. + + :param stack: stack to call a webhook on + :param output_key: The name of the output to get the URL from + :raises: InvalidConfigException if the output key is not found + """ + url = None + for output in stack.outputs: + if output["output_key"] == output_key: + url = output["output_value"] + break + else: + raise exceptions.InvalidConfigException( + "No output key %(key)s found in stack %(id)s" % + {"key": output_key, "id": stack.id}) + + with base.AtomicAction(self, "heat.%s_webhook" % output_key): + requests.post(url).raise_for_status() diff --git a/samples/tasks/scenarios/heat/create-stack-and-scale.json b/samples/tasks/scenarios/heat/create-stack-and-scale.json new file mode 100644 index 00000000..4afcf59f --- /dev/null +++ b/samples/tasks/scenarios/heat/create-stack-and-scale.json @@ -0,0 +1,22 @@ +{ + "HeatStacks.create_stack_and_scale": [ + { + "args": { + "template_path": "templates/autoscaling_group.yaml.template", + "output_key": "scaling_url", + "delta": 1 + }, + "runner": { + "type": "constant", + "concurrency": 2, + "times": 3 + }, + "context": { + "users": { + "users_per_tenant": 1, + "tenants": 2 + } + } + } + ] +} diff --git a/samples/tasks/scenarios/heat/create-stack-and-scale.yaml b/samples/tasks/scenarios/heat/create-stack-and-scale.yaml new file mode 100644 index 00000000..33cc7fdf --- /dev/null +++ b/samples/tasks/scenarios/heat/create-stack-and-scale.yaml @@ -0,0 +1,15 @@ +--- + HeatStacks.create_stack_and_scale: + - + args: + template_path: "templates/autoscaling_group.yaml.template" + output_key: "scaling_url" + delta: 1 + runner: + type: "constant" + times: 3 + concurrency: 2 + context: + users: + tenants: 2 + users_per_tenant: 1 diff --git a/tests/unit/plugins/openstack/scenarios/heat/test_stacks.py b/tests/unit/plugins/openstack/scenarios/heat/test_stacks.py index 56e54bdc..74e380da 100644 --- a/tests/unit/plugins/openstack/scenarios/heat/test_stacks.py +++ b/tests/unit/plugins/openstack/scenarios/heat/test_stacks.py @@ -143,6 +143,25 @@ class HeatStacksTestCase(test.ScenarioTestCase): self.default_environment) mock__delete_stack.assert_called_once_with(fake_stack) + def test_create_stack_and_scale(self): + heat_scenario = stacks.HeatStacks() + stack = mock.Mock() + heat_scenario._create_stack = mock.Mock(return_value=stack) + heat_scenario._scale_stack = mock.Mock() + + heat_scenario.create_stack_and_scale( + self.default_template, "key", -1, + parameters=self.default_parameters, + files=self.default_files, + environment=self.default_environment) + heat_scenario._create_stack.assert_called_once_with( + self.default_template, + self.default_parameters, + self.default_files, + self.default_environment) + heat_scenario._scale_stack.assert_called_once_with( + stack, "key", -1) + @mock.patch(HEAT_STACKS + "._delete_stack") @mock.patch(HEAT_STACKS + "._resume_stack") @mock.patch(HEAT_STACKS + "._suspend_stack") diff --git a/tests/unit/plugins/openstack/scenarios/heat/test_utils.py b/tests/unit/plugins/openstack/scenarios/heat/test_utils.py index bcbc36c2..09110166 100644 --- a/tests/unit/plugins/openstack/scenarios/heat/test_utils.py +++ b/tests/unit/plugins/openstack/scenarios/heat/test_utils.py @@ -191,6 +191,60 @@ class HeatScenarioTestCase(test.ScenarioTestCase): self._test_atomic_action_timer(scenario.atomic_actions(), "heat.restore_stack") + def test__count_instances(self): + self.clients("heat").resources.list.return_value = [ + mock.Mock(resource_type="OS::Nova::Server"), + mock.Mock(resource_type="OS::Nova::Server"), + mock.Mock(resource_type="OS::Heat::AutoScalingGroup")] + scenario = utils.HeatScenario() + self.assertEqual(scenario._count_instances(self.stack), 2) + self.clients("heat").resources.list.assert_called_once_with( + self.stack.id, + nested_depth=1) + + def test__scale_stack(self): + scenario = utils.HeatScenario() + scenario._count_instances = mock.Mock(side_effect=[3, 3, 2]) + scenario._stack_webhook = mock.Mock() + + scenario._scale_stack(self.stack, "test_output_key", -1) + + scenario._stack_webhook.assert_called_once_with(self.stack, + "test_output_key") + self.mock_wait_for.mock.assert_called_once_with( + self.stack, + is_ready=mock.ANY, + update_resource=self.mock_get_from_manager.mock.return_value, + timeout=CONF.benchmark.heat_stack_scale_timeout, + check_interval=CONF.benchmark.heat_stack_scale_poll_interval) + self.mock_get_from_manager.mock.assert_called_once_with( + ["UPDATE_FAILED"]) + + self._test_atomic_action_timer(scenario.atomic_actions(), + "heat.scale_with_test_output_key") + + @mock.patch("requests.post") + def test_stack_webhook(self, mock_post): + scenario = utils.HeatScenario() + stack = mock.Mock(outputs=[ + {"output_key": "output1", "output_value": "url1"}, + {"output_key": "output2", "output_value": "url2"}]) + + scenario._stack_webhook(stack, "output1") + mock_post.assert_called_with("url1") + self._test_atomic_action_timer(scenario.atomic_actions(), + "heat.output1_webhook") + + @mock.patch("requests.post") + def test_stack_webhook_invalid_output_key(self, mock_post): + scenario = utils.HeatScenario() + stack = mock.Mock() + stack.outputs = [{"output_key": "output1", "output_value": "url1"}, + {"output_key": "output2", "output_value": "url2"}] + + self.assertRaises(exceptions.InvalidConfigException, + scenario._stack_webhook, stack, "bogus") + class HeatScenarioNegativeTestCase(test.ScenarioTestCase): patch_benchmark_utils = False