From 48f895b64552b3f8d854868bbefd1ffba8c1ce22 Mon Sep 17 00:00:00 2001 From: Peter Razumovsky Date: Wed, 18 Apr 2018 13:34:50 +0400 Subject: [PATCH] Add Grafana plugin for monitoring testing Add testing monitoring system with push acceptor Pushgateway and metric analyzer Grafana in conjunction with nova instance. This test case is to check how nova instance could communicate with a monitoring system, not only for testing monitoring availability. Suchwise Openstack platform using is justified. However, if the issue is to check availability of monitoring system, push_metric_locally scenario allows to push random metric locally to Pushgateway and just check it in Grafana. Change-Id: I00e85189b5f54c3e6fd18fa52afe7db0eda88fed --- rally_openstack/scenarios/grafana/__init__.py | 0 rally_openstack/scenarios/grafana/metrics.py | 155 ++++++++++++++++++ rally_openstack/services/grafana/__init__.py | 0 rally_openstack/services/grafana/grafana.py | 91 ++++++++++ .../grafana/push-metric-from-instance.json | 58 +++++++ .../grafana/push-metric-from-instance.yaml | 43 +++++ .../grafana/push-metric-locally.json | 35 ++++ .../grafana/push-metric-locally.yaml | 25 +++ 8 files changed, 407 insertions(+) create mode 100644 rally_openstack/scenarios/grafana/__init__.py create mode 100644 rally_openstack/scenarios/grafana/metrics.py create mode 100644 rally_openstack/services/grafana/__init__.py create mode 100644 rally_openstack/services/grafana/grafana.py create mode 100644 samples/tasks/scenarios/grafana/push-metric-from-instance.json create mode 100644 samples/tasks/scenarios/grafana/push-metric-from-instance.yaml create mode 100644 samples/tasks/scenarios/grafana/push-metric-locally.json create mode 100644 samples/tasks/scenarios/grafana/push-metric-locally.yaml diff --git a/rally_openstack/scenarios/grafana/__init__.py b/rally_openstack/scenarios/grafana/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/rally_openstack/scenarios/grafana/metrics.py b/rally_openstack/scenarios/grafana/metrics.py new file mode 100644 index 00000000..bfffb5be --- /dev/null +++ b/rally_openstack/scenarios/grafana/metrics.py @@ -0,0 +1,155 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from rally.common import cfg +from rally.common import logging +from rally.task import types +from rally.task import utils +from rally.task import validation + +from rally_openstack import consts +from rally_openstack import scenario +from rally_openstack.services.grafana import grafana as grafana_service + +CONF = cfg.CONF +LOG = logging.getLogger(__name__) + +"""Scenarios for Pushgateway and Grafana metrics.""" + + +@types.convert(image={"type": "glance_image"}, + flavor={"type": "nova_flavor"}) +@validation.add("required_services", services=[consts.Service.NOVA]) +@validation.add("required_platform", platform="openstack", admin=True) +@scenario.configure(context={"cleanup@openstack": ["nova"]}, + name="GrafanaMetrics.push_metric_from_instance", + platform="openstack") +class PushMetricsInstance(scenario.OpenStackScenario): + """Test monitoring system by pushing metric from nova server and check it. + + Scenario tests monitoring system, which uses Pushgateway as metric exporter + and Grafana as metrics monitoring. + + The goal of the test is to check that monitoring system works correctly + with nova instance. Test case is the following: we deploy some env with + nodes on Openstack nova instances, add metric exporter (using Pushgateway + in this test) inside nodes (i.e. nova instances) for some interested + metrics (e.g. CPU, memory etc.). We want to check that metrics successfully + sends to metrics storage (e.g. Prometheus) by requesting Grafana. Create + nova instance, add Pushgateway push random metric to userdata and after + instance would be available, check Grafana datasource that pushed metric in + data. + """ + + def _metric_from_instance(self, seed, image, flavor, monitor_vip, + pushgateway_port, job_name): + push_cmd = ( + "echo %(seed)s 12345 | curl --data-binary " + "@- http://%(monitor_vip)s:%(pgtw_port)s/metrics/job" + "/%(job_name)s" % {"seed": seed, + "monitor_vip": monitor_vip, + "pgtw_port": pushgateway_port, + "job_name": job_name}) + userdata = ("#!/bin/bash\n%s" % push_cmd) + server = self.clients("nova").servers.create(seed, + image, flavor, + userdata=userdata) + LOG.info("Server %s create started" % seed) + self.sleep_between(CONF.openstack.nova_server_boot_prepoll_delay) + utils.wait_for_status( + server, + ready_statuses=["ACTIVE"], + update_resource=utils.get_from_manager(), + timeout=CONF.openstack.nova_server_boot_timeout, + check_interval=CONF.openstack.nova_server_boot_poll_interval + ) + LOG.info("Server %s with pushing metric script (metric exporter) is " + "active" % seed) + + def run(self, image, flavor, monitor_vip, pushgateway_port, + grafana, datasource_id, job_name, sleep_time=5, + retries_total=30): + """Create nova instance with pushing metric script as userdata. + + Push metric to metrics storage using Pushgateway and check it in + Grafana. + + :param image: image for server with userdata script + :param flavor: flavor for server with userdata script + :param monitor_vip: monitoring system IP to push metric + :param pushgateway_port: Pushgateway port to use for pushing metric + :param grafana: Grafana dict with creds and port to use for checking + metric. Format: {user: admin, password: pass, port: 9902} + :param datasource_id: metrics storage datasource ID in Grafana + :param job_name: job name to push metric in it + :param sleep_time: sleep time between checking metrics in seconds + :param retries_total: total number of retries to check metric in + Grafana + """ + seed = self.generate_random_name() + + grafana_svc = grafana_service.GrafanaService( + dict(monitor_vip=monitor_vip, pushgateway_port=pushgateway_port, + grafana=grafana, datasource_id=datasource_id, + job_name=job_name), + name_generator=self.generate_random_name, + atomic_inst=self.atomic_actions()) + + self._metric_from_instance(seed, image, flavor, monitor_vip, + pushgateway_port, job_name) + checked = grafana_svc.check_metric(seed, monitor_vip=monitor_vip, + grafana=grafana, + datasource_id=datasource_id, + sleep_time=sleep_time, + retries_total=retries_total) + self.assertTrue(checked) + + +@scenario.configure(name="GrafanaMetrics.push_metric_locally") +class PushMetricLocal(scenario.OpenStackScenario): + """Test monitoring system availability with local pushing random metric.""" + + def run(self, monitor_vip, pushgateway_port, grafana, datasource_id, + job_name, sleep_time=5, retries_total=30): + """Push random metric to Pushgateway locally and check it in Grafana. + + :param monitor_vip: monitoring system IP to push metric + :param pushgateway_port: Pushgateway port to use for pushing metric + :param grafana: Grafana dict with creds and port to use for checking + metric. Format: {user: admin, password: pass, port: 9902} + :param datasource_id: metrics storage datasource ID in Grafana + :param job_name: job name to push metric in it + :param sleep_time: sleep time between checking metrics in seconds + :param retries_total: total number of retries to check metric in + Grafana + """ + seed = self.generate_random_name() + + grafana_svc = grafana_service.GrafanaService( + dict(monitor_vip=monitor_vip, pushgateway_port=pushgateway_port, + grafana=grafana, datasource_id=datasource_id, + job_name=job_name), + name_generator=self.generate_random_name, + atomic_inst=self.atomic_actions()) + + pushed = grafana_svc.push_metric(seed, monitor_vip=monitor_vip, + pushgateway_port=pushgateway_port, + job_name=job_name) + self.assertTrue(pushed) + checked = grafana_svc.check_metric(seed, monitor_vip=monitor_vip, + grafana=grafana, + datasource_id=datasource_id, + sleep_time=sleep_time, + retries_total=retries_total) + self.assertTrue(checked) diff --git a/rally_openstack/services/grafana/__init__.py b/rally_openstack/services/grafana/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/rally_openstack/services/grafana/grafana.py b/rally_openstack/services/grafana/grafana.py new file mode 100644 index 00000000..71eb9a16 --- /dev/null +++ b/rally_openstack/services/grafana/grafana.py @@ -0,0 +1,91 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import requests + +from rally.common import logging +from rally.common import utils as commonutils +from rally.task import atomic +from rally.task import service + +LOG = logging.getLogger(__name__) + + +class GrafanaService(service.Service): + + def __init__(self, spec, name_generator=None, atomic_inst=None): + """Initialization of Grafana service. + + :param spec: param contains monitoring system info: IPs, ports, creds + """ + super(GrafanaService, self).__init__(None, + name_generator=name_generator, + atomic_inst=atomic_inst) + self._spec = spec + + @atomic.action_timer("grafana.check_metric") + def check_metric(self, seed, sleep_time, retries_total): + """Check metric with seed name in Grafana datasource. + + :param seed: random metric name + :param sleep_time: sleep time between checking metrics in seconds + :param retries_total: total number of retries to check metric in + Grafana + :return: True if metric in Grafana datasource and False otherwise + """ + check_url = ("http://%(vip)s:%(port)s/api/datasources/proxy/:" + "%(datasource)s/api/v1/query?query=%(seed)s" % { + "vip": self._spec["monitor_vip"], + "port": self._spec["grafana"]["port"], + "datasource": self._spec["datasource_id"], + "seed": seed + }) + i = 0 + LOG.info("Check metric %s in Grafana" % seed) + while i < retries_total: + LOG.debug("Attempt number %s" % (i + 1)) + resp = requests.get(check_url, + auth=(self._spec["grafana"]["user"], + self._spec["grafana"]["password"])) + result = resp.json() + LOG.debug("Grafana response code: %s" % resp.status_code) + if len(result["data"]["result"]) < 1 and i + 1 >= retries_total: + LOG.debug("No instance metrics found in Grafana") + return False + elif len(result["data"]["result"]) < 1: + i += 1 + commonutils.interruptable_sleep(sleep_time) + else: + LOG.debug("Metric instance found in Grafana") + return True + + @atomic.action_timer("grafana.push_metric") + def push_metric(self, seed): + """Push metric by GET request using pushgateway. + + :param seed: random name for metric to push + """ + push_url = "http://%(ip)s:%(port)s/metrics/job/%(job)s" % { + "ip": self._spec["monitor_vip"], + "port": self._spec["pushgateway_port"], + "job": self._spec["job_name"] + } + resp = requests.post(push_url, + headers={"Content-type": "text/xml"}, + data="%s 12345\n" % seed) + if resp.ok: + LOG.info("Metric %s pushed" % seed) + else: + LOG.error("Error during push metric %s" % seed) + return resp.ok diff --git a/samples/tasks/scenarios/grafana/push-metric-from-instance.json b/samples/tasks/scenarios/grafana/push-metric-from-instance.json new file mode 100644 index 00000000..e7acd8f0 --- /dev/null +++ b/samples/tasks/scenarios/grafana/push-metric-from-instance.json @@ -0,0 +1,58 @@ +{% set flavor_name = flavor_name or "grafana_test.small" %} +{% set image_name = image_name or "testVM" %} +{ + "GrafanaMetrics.push_metric_from_instance": [ + { + "args": { + "flavor": { + "name": "{{ flavor_name }}" + }, + "image": { + "name": "{{ image_name }}" + }, + "monitor_vip": "10.0.0.5", + "pushgateway_port": 9091, + "grafana": { + "user": "admin", + "password": "password", + "port": 3000 + }, + "datasource_id": 1, + "job_name": "rally_test", + "sleep_time": 5, + "retries_total": 30 + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 1 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + }, + "flavors": [ + { + "name": "{{ flavor_name }}", + "ram": 512, + "disk": 1, + "vcpus": 1 + } + ], + "images": { + "image_name": "{{ image_name }}", + "image_url": "http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img", + "disk_format": "qcow2", + "container_format": "bare", + "visibility": "public" + } + }, + "sla": { + "failure_rate": { + "max": 0 + } + } + } + ] +} diff --git a/samples/tasks/scenarios/grafana/push-metric-from-instance.yaml b/samples/tasks/scenarios/grafana/push-metric-from-instance.yaml new file mode 100644 index 00000000..010c9865 --- /dev/null +++ b/samples/tasks/scenarios/grafana/push-metric-from-instance.yaml @@ -0,0 +1,43 @@ +{% set flavor_name = flavor_name or "grafana_test.small" %} +{% set image_name = image_name or "testVM" %} +--- + GrafanaMetrics.push_metric_from_instance: + - + args: + flavor: + name: {{ flavor_name }} + image: + name: {{ image_name }} + monitor_vip: 10.0.0.5 + pushgateway_port: 9091 + grafana: + user: admin + password: password + port: 3000 + datasource_id: 1 + job_name: rally_test + sleep_time: 5 + retries_total: 30 + runner: + type: "constant" + times: 10 + concurrency: 1 + context: + users: + tenants: 1 + users_per_tenant: 1 + flavors: + - + name: {{ flavor_name }} + ram: 512 + disk: 1 + vcpus: 1 + images: + image_name: {{ image_name }} + image_url: http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img + disk_format: qcow2 + container_format: bare + visibility: public + sla: + failure_rate: + max: 0 diff --git a/samples/tasks/scenarios/grafana/push-metric-locally.json b/samples/tasks/scenarios/grafana/push-metric-locally.json new file mode 100644 index 00000000..43b13259 --- /dev/null +++ b/samples/tasks/scenarios/grafana/push-metric-locally.json @@ -0,0 +1,35 @@ +{ + "GrafanaMetrics.push_metric_locally": [ + { + "args": { + "monitor_vip": "10.0.0.5", + "pushgateway_port": 9091, + "grafana": { + "user": "admin", + "password": "password", + "port": 3000 + }, + "datasource_id": 1, + "job_name": "rally_test", + "sleep_time": 5, + "retries_total": 30 + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 1 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + } + }, + "sla": { + "failure_rate": { + "max": 0 + } + } + } + ] +} \ No newline at end of file diff --git a/samples/tasks/scenarios/grafana/push-metric-locally.yaml b/samples/tasks/scenarios/grafana/push-metric-locally.yaml new file mode 100644 index 00000000..c183d901 --- /dev/null +++ b/samples/tasks/scenarios/grafana/push-metric-locally.yaml @@ -0,0 +1,25 @@ +--- + GrafanaMetrics.push_metric_locally: + - + args: + monitor_vip: 10.0.0.5 + pushgateway_port: 9091 + grafana: + user: admin + password: password + port: 3000 + datasource_id: 1 + job_name: rally_test + sleep_time: 5 + retries_total: 30 + runner: + type: "constant" + times: 10 + concurrency: 1 + context: + users: + tenants: 1 + users_per_tenant: 1 + sla: + failure_rate: + max: 0