diff --git a/rally_openstack/scenarios/grafana/__init__.py b/rally_openstack/scenarios/grafana/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/rally_openstack/scenarios/grafana/metrics.py b/rally_openstack/scenarios/grafana/metrics.py new file mode 100644 index 00000000..bfffb5be --- /dev/null +++ b/rally_openstack/scenarios/grafana/metrics.py @@ -0,0 +1,155 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from rally.common import cfg +from rally.common import logging +from rally.task import types +from rally.task import utils +from rally.task import validation + +from rally_openstack import consts +from rally_openstack import scenario +from rally_openstack.services.grafana import grafana as grafana_service + +CONF = cfg.CONF +LOG = logging.getLogger(__name__) + +"""Scenarios for Pushgateway and Grafana metrics.""" + + +@types.convert(image={"type": "glance_image"}, + flavor={"type": "nova_flavor"}) +@validation.add("required_services", services=[consts.Service.NOVA]) +@validation.add("required_platform", platform="openstack", admin=True) +@scenario.configure(context={"cleanup@openstack": ["nova"]}, + name="GrafanaMetrics.push_metric_from_instance", + platform="openstack") +class PushMetricsInstance(scenario.OpenStackScenario): + """Test monitoring system by pushing metric from nova server and check it. + + Scenario tests monitoring system, which uses Pushgateway as metric exporter + and Grafana as metrics monitoring. + + The goal of the test is to check that monitoring system works correctly + with nova instance. Test case is the following: we deploy some env with + nodes on Openstack nova instances, add metric exporter (using Pushgateway + in this test) inside nodes (i.e. nova instances) for some interested + metrics (e.g. CPU, memory etc.). We want to check that metrics successfully + sends to metrics storage (e.g. Prometheus) by requesting Grafana. Create + nova instance, add Pushgateway push random metric to userdata and after + instance would be available, check Grafana datasource that pushed metric in + data. + """ + + def _metric_from_instance(self, seed, image, flavor, monitor_vip, + pushgateway_port, job_name): + push_cmd = ( + "echo %(seed)s 12345 | curl --data-binary " + "@- http://%(monitor_vip)s:%(pgtw_port)s/metrics/job" + "/%(job_name)s" % {"seed": seed, + "monitor_vip": monitor_vip, + "pgtw_port": pushgateway_port, + "job_name": job_name}) + userdata = ("#!/bin/bash\n%s" % push_cmd) + server = self.clients("nova").servers.create(seed, + image, flavor, + userdata=userdata) + LOG.info("Server %s create started" % seed) + self.sleep_between(CONF.openstack.nova_server_boot_prepoll_delay) + utils.wait_for_status( + server, + ready_statuses=["ACTIVE"], + update_resource=utils.get_from_manager(), + timeout=CONF.openstack.nova_server_boot_timeout, + check_interval=CONF.openstack.nova_server_boot_poll_interval + ) + LOG.info("Server %s with pushing metric script (metric exporter) is " + "active" % seed) + + def run(self, image, flavor, monitor_vip, pushgateway_port, + grafana, datasource_id, job_name, sleep_time=5, + retries_total=30): + """Create nova instance with pushing metric script as userdata. + + Push metric to metrics storage using Pushgateway and check it in + Grafana. + + :param image: image for server with userdata script + :param flavor: flavor for server with userdata script + :param monitor_vip: monitoring system IP to push metric + :param pushgateway_port: Pushgateway port to use for pushing metric + :param grafana: Grafana dict with creds and port to use for checking + metric. Format: {user: admin, password: pass, port: 9902} + :param datasource_id: metrics storage datasource ID in Grafana + :param job_name: job name to push metric in it + :param sleep_time: sleep time between checking metrics in seconds + :param retries_total: total number of retries to check metric in + Grafana + """ + seed = self.generate_random_name() + + grafana_svc = grafana_service.GrafanaService( + dict(monitor_vip=monitor_vip, pushgateway_port=pushgateway_port, + grafana=grafana, datasource_id=datasource_id, + job_name=job_name), + name_generator=self.generate_random_name, + atomic_inst=self.atomic_actions()) + + self._metric_from_instance(seed, image, flavor, monitor_vip, + pushgateway_port, job_name) + checked = grafana_svc.check_metric(seed, monitor_vip=monitor_vip, + grafana=grafana, + datasource_id=datasource_id, + sleep_time=sleep_time, + retries_total=retries_total) + self.assertTrue(checked) + + +@scenario.configure(name="GrafanaMetrics.push_metric_locally") +class PushMetricLocal(scenario.OpenStackScenario): + """Test monitoring system availability with local pushing random metric.""" + + def run(self, monitor_vip, pushgateway_port, grafana, datasource_id, + job_name, sleep_time=5, retries_total=30): + """Push random metric to Pushgateway locally and check it in Grafana. + + :param monitor_vip: monitoring system IP to push metric + :param pushgateway_port: Pushgateway port to use for pushing metric + :param grafana: Grafana dict with creds and port to use for checking + metric. Format: {user: admin, password: pass, port: 9902} + :param datasource_id: metrics storage datasource ID in Grafana + :param job_name: job name to push metric in it + :param sleep_time: sleep time between checking metrics in seconds + :param retries_total: total number of retries to check metric in + Grafana + """ + seed = self.generate_random_name() + + grafana_svc = grafana_service.GrafanaService( + dict(monitor_vip=monitor_vip, pushgateway_port=pushgateway_port, + grafana=grafana, datasource_id=datasource_id, + job_name=job_name), + name_generator=self.generate_random_name, + atomic_inst=self.atomic_actions()) + + pushed = grafana_svc.push_metric(seed, monitor_vip=monitor_vip, + pushgateway_port=pushgateway_port, + job_name=job_name) + self.assertTrue(pushed) + checked = grafana_svc.check_metric(seed, monitor_vip=monitor_vip, + grafana=grafana, + datasource_id=datasource_id, + sleep_time=sleep_time, + retries_total=retries_total) + self.assertTrue(checked) diff --git a/rally_openstack/services/grafana/__init__.py b/rally_openstack/services/grafana/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/rally_openstack/services/grafana/grafana.py b/rally_openstack/services/grafana/grafana.py new file mode 100644 index 00000000..71eb9a16 --- /dev/null +++ b/rally_openstack/services/grafana/grafana.py @@ -0,0 +1,91 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import requests + +from rally.common import logging +from rally.common import utils as commonutils +from rally.task import atomic +from rally.task import service + +LOG = logging.getLogger(__name__) + + +class GrafanaService(service.Service): + + def __init__(self, spec, name_generator=None, atomic_inst=None): + """Initialization of Grafana service. + + :param spec: param contains monitoring system info: IPs, ports, creds + """ + super(GrafanaService, self).__init__(None, + name_generator=name_generator, + atomic_inst=atomic_inst) + self._spec = spec + + @atomic.action_timer("grafana.check_metric") + def check_metric(self, seed, sleep_time, retries_total): + """Check metric with seed name in Grafana datasource. + + :param seed: random metric name + :param sleep_time: sleep time between checking metrics in seconds + :param retries_total: total number of retries to check metric in + Grafana + :return: True if metric in Grafana datasource and False otherwise + """ + check_url = ("http://%(vip)s:%(port)s/api/datasources/proxy/:" + "%(datasource)s/api/v1/query?query=%(seed)s" % { + "vip": self._spec["monitor_vip"], + "port": self._spec["grafana"]["port"], + "datasource": self._spec["datasource_id"], + "seed": seed + }) + i = 0 + LOG.info("Check metric %s in Grafana" % seed) + while i < retries_total: + LOG.debug("Attempt number %s" % (i + 1)) + resp = requests.get(check_url, + auth=(self._spec["grafana"]["user"], + self._spec["grafana"]["password"])) + result = resp.json() + LOG.debug("Grafana response code: %s" % resp.status_code) + if len(result["data"]["result"]) < 1 and i + 1 >= retries_total: + LOG.debug("No instance metrics found in Grafana") + return False + elif len(result["data"]["result"]) < 1: + i += 1 + commonutils.interruptable_sleep(sleep_time) + else: + LOG.debug("Metric instance found in Grafana") + return True + + @atomic.action_timer("grafana.push_metric") + def push_metric(self, seed): + """Push metric by GET request using pushgateway. + + :param seed: random name for metric to push + """ + push_url = "http://%(ip)s:%(port)s/metrics/job/%(job)s" % { + "ip": self._spec["monitor_vip"], + "port": self._spec["pushgateway_port"], + "job": self._spec["job_name"] + } + resp = requests.post(push_url, + headers={"Content-type": "text/xml"}, + data="%s 12345\n" % seed) + if resp.ok: + LOG.info("Metric %s pushed" % seed) + else: + LOG.error("Error during push metric %s" % seed) + return resp.ok diff --git a/samples/tasks/scenarios/grafana/push-metric-from-instance.json b/samples/tasks/scenarios/grafana/push-metric-from-instance.json new file mode 100644 index 00000000..e7acd8f0 --- /dev/null +++ b/samples/tasks/scenarios/grafana/push-metric-from-instance.json @@ -0,0 +1,58 @@ +{% set flavor_name = flavor_name or "grafana_test.small" %} +{% set image_name = image_name or "testVM" %} +{ + "GrafanaMetrics.push_metric_from_instance": [ + { + "args": { + "flavor": { + "name": "{{ flavor_name }}" + }, + "image": { + "name": "{{ image_name }}" + }, + "monitor_vip": "10.0.0.5", + "pushgateway_port": 9091, + "grafana": { + "user": "admin", + "password": "password", + "port": 3000 + }, + "datasource_id": 1, + "job_name": "rally_test", + "sleep_time": 5, + "retries_total": 30 + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 1 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + }, + "flavors": [ + { + "name": "{{ flavor_name }}", + "ram": 512, + "disk": 1, + "vcpus": 1 + } + ], + "images": { + "image_name": "{{ image_name }}", + "image_url": "http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img", + "disk_format": "qcow2", + "container_format": "bare", + "visibility": "public" + } + }, + "sla": { + "failure_rate": { + "max": 0 + } + } + } + ] +} diff --git a/samples/tasks/scenarios/grafana/push-metric-from-instance.yaml b/samples/tasks/scenarios/grafana/push-metric-from-instance.yaml new file mode 100644 index 00000000..010c9865 --- /dev/null +++ b/samples/tasks/scenarios/grafana/push-metric-from-instance.yaml @@ -0,0 +1,43 @@ +{% set flavor_name = flavor_name or "grafana_test.small" %} +{% set image_name = image_name or "testVM" %} +--- + GrafanaMetrics.push_metric_from_instance: + - + args: + flavor: + name: {{ flavor_name }} + image: + name: {{ image_name }} + monitor_vip: 10.0.0.5 + pushgateway_port: 9091 + grafana: + user: admin + password: password + port: 3000 + datasource_id: 1 + job_name: rally_test + sleep_time: 5 + retries_total: 30 + runner: + type: "constant" + times: 10 + concurrency: 1 + context: + users: + tenants: 1 + users_per_tenant: 1 + flavors: + - + name: {{ flavor_name }} + ram: 512 + disk: 1 + vcpus: 1 + images: + image_name: {{ image_name }} + image_url: http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img + disk_format: qcow2 + container_format: bare + visibility: public + sla: + failure_rate: + max: 0 diff --git a/samples/tasks/scenarios/grafana/push-metric-locally.json b/samples/tasks/scenarios/grafana/push-metric-locally.json new file mode 100644 index 00000000..43b13259 --- /dev/null +++ b/samples/tasks/scenarios/grafana/push-metric-locally.json @@ -0,0 +1,35 @@ +{ + "GrafanaMetrics.push_metric_locally": [ + { + "args": { + "monitor_vip": "10.0.0.5", + "pushgateway_port": 9091, + "grafana": { + "user": "admin", + "password": "password", + "port": 3000 + }, + "datasource_id": 1, + "job_name": "rally_test", + "sleep_time": 5, + "retries_total": 30 + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 1 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + } + }, + "sla": { + "failure_rate": { + "max": 0 + } + } + } + ] +} \ No newline at end of file diff --git a/samples/tasks/scenarios/grafana/push-metric-locally.yaml b/samples/tasks/scenarios/grafana/push-metric-locally.yaml new file mode 100644 index 00000000..c183d901 --- /dev/null +++ b/samples/tasks/scenarios/grafana/push-metric-locally.yaml @@ -0,0 +1,25 @@ +--- + GrafanaMetrics.push_metric_locally: + - + args: + monitor_vip: 10.0.0.5 + pushgateway_port: 9091 + grafana: + user: admin + password: password + port: 3000 + datasource_id: 1 + job_name: rally_test + sleep_time: 5 + retries_total: 30 + runner: + type: "constant" + times: 10 + concurrency: 1 + context: + users: + tenants: 1 + users_per_tenant: 1 + sla: + failure_rate: + max: 0