Add option to create Grafana annotations

This patch adds an option in browbeat-config.yaml to create annotations
on a Grafana dashboard for a Browbeat scenario. This would be useful for
CI as it provides information on Grafana about what Browbeat scenario was
running at a particular time.

Change-Id: I83a9c74a56379da35ec9466a7492aecc2ee64ea9

ansible/install/group_vars/all.yml

@@ -459,6 +459,10 @@ dashboards_batch: 20
 # For use with all-{cpu, memory, disk, network} openstack dashboards, uses the graphite prefix to create dashboards for specific openstack cloud
 dashboard_cloud_name: "{{graphite_prefix}}"
 
+# Dashboard UID to create Grafana annotations for Browbeat/Rally scenarios.
+# This is optional unless create_grafana_annotations is enabled in browbeat-config.yaml.
+grafana_dashboard_uid:
+
 ########################################
 # StatsD Configuration
 # Points at configured Graphite instance

browbeat-complete.yaml

@@ -10,6 +10,7 @@ browbeat:
   # "cd ansible;ansible-playbook -i hosts.yml -vvv install/collectd.yml" before
   # setting this option to true.
   start_stop_collectd: false
+  create_grafana_annotations: false
 ansible:
   hosts: ansible/hosts
   metadata_playbook: ansible/gather/stockpile.yml

browbeat-config.yaml

@@ -14,6 +14,17 @@ browbeat:
   # "cd ansible;ansible-playbook -i hosts.yml -vvv install/collectd.yml" before
   # setting this option to true.
   start_stop_collectd: true
+  # This option enables creation of annotations on the Grafana dashboard.
+  # Separate annotations will be created on all panels for the duration of
+  # each scenario that is run using this browbeat configuration file.
+  # grafana_host, grafana_port, grafana_username, grafana_password
+  # and grafana_dashboard_uid have to be passed in
+  # ansible/install/group_vars/all.yml before this option is enabled.
+  # In the Openstack General System Performance Dashboard, the default
+  # annotation setting should be set to query by tag $Cloud when this feature
+  # is enabled.
+  # This feature has been tested on Grafana v9.2.0
+  create_grafana_annotations: false
 ansible:
   hosts: ansible/hosts.yml
   metadata_playbook: ansible/gather/stockpile.yml

browbeat/grafana.py

@@ -11,7 +11,10 @@
 #   limitations under the License.
 
 import logging
-
+import yaml
+import json
+import requests
+from requests.auth import HTTPBasicAuth
 
 class Grafana(object):
 
@@ -47,3 +50,57 @@ class Grafana(object):
                     test_name,
                     dashboard,
                     self.grafana_url[dashboard]))
+
+    def create_grafana_annotations(self, start_timestamp, end_timestamp, scenario_name,
+                                   test_name, times, concurrency):
+        """Create annotations on Grafana dashboard for a given scenario
+        :param start_timestamp: epoch time when scenario started running
+        :param end_timestamp: epoch time when scenario finished running
+        :param scenario_name: str, name of Browbeat scenario
+        :param test_name: str, name of test in Rally DB
+        :param times: int, times value from Rally
+        :param concurrency: int, concurrency value from Rally
+        """
+        with open("ansible/install/group_vars/all.yml", "r") as group_vars_file:
+            group_vars = yaml.safe_load(group_vars_file)
+
+        required_fields = ["grafana_host", "grafana_port", "grafana_username",
+                           "grafana_dashboard_uid", "cloud_prefix"]
+
+        for field in required_fields:
+            if group_vars[field] is None:
+                raise Exception("""{} in ansible/install/group_vars/all.yml is
+                                required to create grafana annotations""".format(
+                                field))
+
+        grafana_host = group_vars["grafana_host"]
+        grafana_port = group_vars["grafana_port"]
+        grafana_username = group_vars["grafana_username"]
+        grafana_password = group_vars["grafana_password"]
+        grafana_dashboard_uid = group_vars["grafana_dashboard_uid"]
+        cloud_prefix = group_vars["cloud_prefix"]
+
+        request_body = {"dashboardUID": grafana_dashboard_uid, "time": start_timestamp,
+                        "timeEnd": end_timestamp,
+                        "tags": [cloud_prefix,
+                                 "times: {}".format(times),
+                                 "concurrency: {}".format(concurrency),
+                                 # test_name_prefix contains a time in the
+                                 # yyyymmdd-hhmmss format. This prefix can
+                                 # be used to locate Browbeat results of runs easily.
+                                 "test_name_prefix: {}".format("-".join(test_name.split("-")[:2]))],
+                        "text": scenario_name}
+
+        headers = {'Content-type':'application/json', 'Accept':'application/json'}
+
+        response = requests.post(url="http://{}:{}/api/annotations".format(
+                                 grafana_host, grafana_port), data=json.dumps(request_body),
+                                 auth=HTTPBasicAuth(grafana_username, grafana_password),
+                                 headers=headers)
+
+        if response.status_code == 200:
+            self.logger.info("Grafana annotation created successfully for scenario {}".format(
+                             scenario_name))
+        else:
+            self.logger.warning("Grafana annotation creation failed : {}".format(
+                                response.json()))

browbeat/schema/browbeat.yml

@@ -22,6 +22,9 @@ mapping:
       start_stop_collectd:
         type: bool
         required: True
+      create_grafana_annotations:
+        type: bool
+        required: True
   ansible:
     required: True
     type: map

browbeat/tools.py

@@ -98,7 +98,8 @@ class Tools(object):
             .format(self.config['ansible']['check_collectd_config_playbook'])
         returncode = self.run_cmd(ansible_cmd)['rc']
         if returncode > 0:
-            self.logger.warning("""graphite_host is empty in all.yml. Please fill it and run the command
+            self.logger.warning("""graphite_host is empty in all.yml. Please fill it and
+                                run the command
                                 (cd ansible;ansible-playbook -i hosts.yml -vvv install/collectd.yml)
                                 in order to install collectd.""")
             return False

browbeat/workloads/rally.py

@@ -37,7 +37,8 @@ class Rally(base.WorkloadBase):
         self.grafana = grafana.Grafana(self.config)
         self.elastic = elastic.Elastic(self.config, self.__class__.__name__.lower())
 
-    def run_scenario(self, task_file, scenario_args, result_dir, test_name, benchmark):
+    def run_scenario(self, task_file, scenario_args, result_dir, test_name, benchmark,
+                     scenario_name):
         self.logger.debug("--------------------------------")
         self.logger.debug("task_file: {}".format(task_file))
         self.logger.info("Running with scenario_args: {}".format(json.dumps(scenario_args)))
@@ -69,6 +70,10 @@ class Rally(base.WorkloadBase):
         to_ts = int(time.time() * 1000)
         self.grafana.create_grafana_urls({'from_ts': from_ts, 'to_ts': to_ts})
         self.grafana.print_dashboard_url(test_name)
+        if self.config["browbeat"]["create_grafana_annotations"]:
+            self.grafana.create_grafana_annotations(from_ts, to_ts, scenario_name,
+                                                    test_name, scenario_args["times"],
+                                                    scenario_args["concurrency"])
         return (from_time, to_time)
 
     def get_task_id(self, test_name):
@@ -286,7 +291,8 @@ class Rally(base.WorkloadBase):
                         exit(1)
 
                     from_time, to_time = self.run_scenario(
-                        scenario_file, scenario, result_dir, test_name, workload["name"])
+                        scenario_file, scenario, result_dir, test_name, workload["name"],
+                        scenario_name)
 
                     new_test_name = test_name.split("-")
                     new_test_name = new_test_name[3:]

conf/browbeat-glance-complete.yaml

@@ -4,6 +4,7 @@ browbeat:
   rerun: 1
   rerun_type: iteration
   start_stop_collectd: false
+  create_grafana_annotations: false
 ansible:
   hosts: ansible/hosts
   metadata_playbook: ansible/gather/stockpile.yml

conf/browbeat-keystone-complete.yaml

@@ -5,6 +5,7 @@ browbeat:
   rerun: 3
   rerun_type: iteration
   start_stop_collectd: false
+  create_grafana_annotations: false
 ansible:
   hosts: ansible/hosts
   metadata_playbook: ansible/gather/stockpile.yml

conf/browbeat-keystone-minimal.yaml

@@ -5,6 +5,7 @@ browbeat:
   rerun: 1
   rerun_type: iteration
   start_stop_collectd: false
+  create_grafana_annotations: false
 ansible:
   hosts: ansible/hosts
   metadata_playbook: ansible/gather/stockpile.yml

conf/browbeat-workloads.yml

@@ -4,6 +4,7 @@ browbeat:
   rerun: 1
   rerun_type: iteration
   start_stop_collectd: false
+  create_grafana_annotations: false
 ansible:
   hosts: ansible/hosts
   metadata_playbook: ansible/gather/stockpile.yml

conf/ceilometer-minimal.yaml

@@ -4,6 +4,7 @@ browbeat:
   rerun: 1
   rerun_type: iteration
   start_stop_collectd: false
+  create_grafana_annotations: false
 ansible:
   hosts: ansible/hosts
   metadata_playbook: ansible/gather/stockpile.yml

conf/ceilometer-persist-response-timing.yaml

@@ -7,6 +7,7 @@ browbeat:
   rerun: 10
   rerun_type: complete
   start_stop_collectd: false
+  create_grafana_annotations: false
 ansible:
   hosts: ansible/hosts
   metadata_playbook: ansible/gather/stockpile.yml

conf/gnocchi-minimal.yaml

@@ -5,6 +5,7 @@ browbeat:
   rerun: 1
   rerun_type: iteration
   start_stop_collectd: false
+  create_grafana_annotations: false
 ansible:
   hosts: ansible/hosts
   metadata_playbook: ansible/gather/stockpile.yml

conf/quickstart.yml

@@ -4,6 +4,7 @@ browbeat:
   rerun: 1
   rerun_type: iteration
   start_stop_collectd: false
+  create_grafana_annotations: false
 ansible:
   hosts: ansible/hosts
   metadata_playbook: ansible/gather/stockpile.yml

conf/telemetry-nova-neutron-gnocchi-10k-1000-per-30m.yaml

@@ -11,6 +11,7 @@ browbeat:
   rerun: 10
   rerun_type: complete
   start_stop_collectd: false
+  create_grafana_annotations: false
 ansible:
   hosts: ansible/hosts
   metadata_playbook: ansible/gather/stockpile.yml

conf/telemetry-nova-neutron-gnocchi-1k-100-per-30m.yaml

@@ -11,6 +11,7 @@ browbeat:
   rerun: 10
   rerun_type: complete
   start_stop_collectd: false
+  create_grafana_annotations: false
 ansible:
   hosts: ansible/hosts
   metadata_playbook: ansible/gather/stockpile.yml

conf/telemetry-nova-neutron-gnocchi-1k-500-per-30m.yaml

@@ -11,6 +11,7 @@ browbeat:
   rerun: 2
   rerun_type: complete
   start_stop_collectd: false
+  create_grafana_annotations: false
 ansible:
   hosts: ansible/hosts
   metadata_playbook: ansible/gather/stockpile.yml

conf/telemetry-nova-neutron-gnocchi-5k-500-per-30m.yaml

@@ -11,6 +11,7 @@ browbeat:
   rerun: 10
   rerun_type: complete
   start_stop_collectd: false
+  create_grafana_annotations: false
 ansible:
   hosts: ansible/hosts
   metadata_playbook: ansible/gather/stockpile.yml

conf/telemetry-nova-persist-instances-with-network-fip.yaml

@@ -8,6 +8,7 @@ browbeat:
   rerun: 1
   rerun_type: iteration
   start_stop_collectd: false
+  create_grafana_annotations: false
 ansible:
   hosts: ansible/hosts
   metadata_playbook: ansible/gather/stockpile.yml

conf/telemetry-nova-persist-instances-with-network-volume-fip.yaml

@@ -8,6 +8,7 @@ browbeat:
   rerun: 1
   rerun_type: iteration
   start_stop_collectd: false
+  create_grafana_annotations: false
 ansible:
   hosts: ansible/hosts
   metadata_playbook: ansible/gather/stockpile.yml

conf/telemetry-nova-persist-instances-with-network-volume.yaml

@@ -7,6 +7,7 @@ browbeat:
   rerun: 1
   rerun_type: iteration
   start_stop_collectd: false
+  create_grafana_annotations: false
 ansible:
   hosts: ansible/hosts
   metadata_playbook: ansible/gather/stockpile.yml

conf/telemetry-nova-persist-instances-with-network.yaml

@@ -9,6 +9,7 @@ browbeat:
   rerun: 1
   rerun_type: iteration
   start_stop_collectd: false
+  create_grafana_annotations: false
 ansible:
   hosts: ansible/hosts
   metadata_playbook: ansible/gather/stockpile.yml

conf/telemetry-nova-persist-instances-with-volume.yaml

@@ -7,6 +7,7 @@ browbeat:
   rerun: 1
   rerun_type: iteration
   start_stop_collectd: false
+  create_grafana_annotations: false
 ansible:
   hosts: ansible/hosts
   metadata_playbook: ansible/gather/stockpile.yml

conf/telemetry-nova-persist-instances.yaml

@@ -8,6 +8,7 @@ browbeat:
   rerun: 1
   rerun_type: iteration
   start_stop_collectd: false
+  create_grafana_annotations: false
 ansible:
   hosts: ansible/hosts
   metadata_playbook: ansible/gather/stockpile.yml

tests/data/invalid_browbeat.yml

@@ -5,6 +5,7 @@ browbeat:
   rerun_type: complete
   invalid_flag: invalid
   start_stop_collectd: false
+  create_grafana_annotations: false
 ansible:
   hosts: ansible/hosts
   metadata_playbook: ansible/gather/site.yml

tests/data/valid_browbeat.yml

@@ -4,6 +4,7 @@ browbeat:
   rerun: 1
   rerun_type: complete
   start_stop_collectd: false
+  create_grafana_annotations: false
 ansible:
   hosts: ansible/hosts
   metadata_playbook: ansible/gather/site.yml