diff --git a/ansible/install/group_vars/all.yml b/ansible/install/group_vars/all.yml index f6143ee54..d338933fb 100644 --- a/ansible/install/group_vars/all.yml +++ b/ansible/install/group_vars/all.yml @@ -459,6 +459,10 @@ dashboards_batch: 20 # For use with all-{cpu, memory, disk, network} openstack dashboards, uses the graphite prefix to create dashboards for specific openstack cloud dashboard_cloud_name: "{{graphite_prefix}}" +# Dashboard UID to create Grafana annotations for Browbeat/Rally scenarios. +# This is optional unless create_grafana_annotations is enabled in browbeat-config.yaml. +grafana_dashboard_uid: + ######################################## # StatsD Configuration # Points at configured Graphite instance diff --git a/browbeat-complete.yaml b/browbeat-complete.yaml index e307963a2..2c4174625 100644 --- a/browbeat-complete.yaml +++ b/browbeat-complete.yaml @@ -10,6 +10,7 @@ browbeat: # "cd ansible;ansible-playbook -i hosts.yml -vvv install/collectd.yml" before # setting this option to true. start_stop_collectd: false + create_grafana_annotations: false ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml diff --git a/browbeat-config.yaml b/browbeat-config.yaml index 0ec7745fb..1f4f6595c 100644 --- a/browbeat-config.yaml +++ b/browbeat-config.yaml @@ -14,6 +14,17 @@ browbeat: # "cd ansible;ansible-playbook -i hosts.yml -vvv install/collectd.yml" before # setting this option to true. start_stop_collectd: true + # This option enables creation of annotations on the Grafana dashboard. + # Separate annotations will be created on all panels for the duration of + # each scenario that is run using this browbeat configuration file. + # grafana_host, grafana_port, grafana_username, grafana_password + # and grafana_dashboard_uid have to be passed in + # ansible/install/group_vars/all.yml before this option is enabled. + # In the Openstack General System Performance Dashboard, the default + # annotation setting should be set to query by tag $Cloud when this feature + # is enabled. + # This feature has been tested on Grafana v9.2.0 + create_grafana_annotations: false ansible: hosts: ansible/hosts.yml metadata_playbook: ansible/gather/stockpile.yml diff --git a/browbeat/grafana.py b/browbeat/grafana.py index b7e3a203e..700d298c2 100644 --- a/browbeat/grafana.py +++ b/browbeat/grafana.py @@ -11,7 +11,10 @@ # limitations under the License. import logging - +import yaml +import json +import requests +from requests.auth import HTTPBasicAuth class Grafana(object): @@ -47,3 +50,57 @@ class Grafana(object): test_name, dashboard, self.grafana_url[dashboard])) + + def create_grafana_annotations(self, start_timestamp, end_timestamp, scenario_name, + test_name, times, concurrency): + """Create annotations on Grafana dashboard for a given scenario + :param start_timestamp: epoch time when scenario started running + :param end_timestamp: epoch time when scenario finished running + :param scenario_name: str, name of Browbeat scenario + :param test_name: str, name of test in Rally DB + :param times: int, times value from Rally + :param concurrency: int, concurrency value from Rally + """ + with open("ansible/install/group_vars/all.yml", "r") as group_vars_file: + group_vars = yaml.safe_load(group_vars_file) + + required_fields = ["grafana_host", "grafana_port", "grafana_username", + "grafana_dashboard_uid", "cloud_prefix"] + + for field in required_fields: + if group_vars[field] is None: + raise Exception("""{} in ansible/install/group_vars/all.yml is + required to create grafana annotations""".format( + field)) + + grafana_host = group_vars["grafana_host"] + grafana_port = group_vars["grafana_port"] + grafana_username = group_vars["grafana_username"] + grafana_password = group_vars["grafana_password"] + grafana_dashboard_uid = group_vars["grafana_dashboard_uid"] + cloud_prefix = group_vars["cloud_prefix"] + + request_body = {"dashboardUID": grafana_dashboard_uid, "time": start_timestamp, + "timeEnd": end_timestamp, + "tags": [cloud_prefix, + "times: {}".format(times), + "concurrency: {}".format(concurrency), + # test_name_prefix contains a time in the + # yyyymmdd-hhmmss format. This prefix can + # be used to locate Browbeat results of runs easily. + "test_name_prefix: {}".format("-".join(test_name.split("-")[:2]))], + "text": scenario_name} + + headers = {'Content-type':'application/json', 'Accept':'application/json'} + + response = requests.post(url="http://{}:{}/api/annotations".format( + grafana_host, grafana_port), data=json.dumps(request_body), + auth=HTTPBasicAuth(grafana_username, grafana_password), + headers=headers) + + if response.status_code == 200: + self.logger.info("Grafana annotation created successfully for scenario {}".format( + scenario_name)) + else: + self.logger.warning("Grafana annotation creation failed : {}".format( + response.json())) diff --git a/browbeat/schema/browbeat.yml b/browbeat/schema/browbeat.yml index 95a7b3303..f8689b7fe 100644 --- a/browbeat/schema/browbeat.yml +++ b/browbeat/schema/browbeat.yml @@ -22,6 +22,9 @@ mapping: start_stop_collectd: type: bool required: True + create_grafana_annotations: + type: bool + required: True ansible: required: True type: map diff --git a/browbeat/tools.py b/browbeat/tools.py index 8e7de972f..c647b6b1a 100644 --- a/browbeat/tools.py +++ b/browbeat/tools.py @@ -98,7 +98,8 @@ class Tools(object): .format(self.config['ansible']['check_collectd_config_playbook']) returncode = self.run_cmd(ansible_cmd)['rc'] if returncode > 0: - self.logger.warning("""graphite_host is empty in all.yml. Please fill it and run the command + self.logger.warning("""graphite_host is empty in all.yml. Please fill it and + run the command (cd ansible;ansible-playbook -i hosts.yml -vvv install/collectd.yml) in order to install collectd.""") return False diff --git a/browbeat/workloads/rally.py b/browbeat/workloads/rally.py index fdae99bb5..e00da1347 100644 --- a/browbeat/workloads/rally.py +++ b/browbeat/workloads/rally.py @@ -37,7 +37,8 @@ class Rally(base.WorkloadBase): self.grafana = grafana.Grafana(self.config) self.elastic = elastic.Elastic(self.config, self.__class__.__name__.lower()) - def run_scenario(self, task_file, scenario_args, result_dir, test_name, benchmark): + def run_scenario(self, task_file, scenario_args, result_dir, test_name, benchmark, + scenario_name): self.logger.debug("--------------------------------") self.logger.debug("task_file: {}".format(task_file)) self.logger.info("Running with scenario_args: {}".format(json.dumps(scenario_args))) @@ -69,6 +70,10 @@ class Rally(base.WorkloadBase): to_ts = int(time.time() * 1000) self.grafana.create_grafana_urls({'from_ts': from_ts, 'to_ts': to_ts}) self.grafana.print_dashboard_url(test_name) + if self.config["browbeat"]["create_grafana_annotations"]: + self.grafana.create_grafana_annotations(from_ts, to_ts, scenario_name, + test_name, scenario_args["times"], + scenario_args["concurrency"]) return (from_time, to_time) def get_task_id(self, test_name): @@ -286,7 +291,8 @@ class Rally(base.WorkloadBase): exit(1) from_time, to_time = self.run_scenario( - scenario_file, scenario, result_dir, test_name, workload["name"]) + scenario_file, scenario, result_dir, test_name, workload["name"], + scenario_name) new_test_name = test_name.split("-") new_test_name = new_test_name[3:] diff --git a/conf/browbeat-glance-complete.yaml b/conf/browbeat-glance-complete.yaml index ae926aefc..30d738cb5 100644 --- a/conf/browbeat-glance-complete.yaml +++ b/conf/browbeat-glance-complete.yaml @@ -4,6 +4,7 @@ browbeat: rerun: 1 rerun_type: iteration start_stop_collectd: false + create_grafana_annotations: false ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml diff --git a/conf/browbeat-keystone-complete.yaml b/conf/browbeat-keystone-complete.yaml index 7c5d4b9da..ad2dbaf22 100644 --- a/conf/browbeat-keystone-complete.yaml +++ b/conf/browbeat-keystone-complete.yaml @@ -5,6 +5,7 @@ browbeat: rerun: 3 rerun_type: iteration start_stop_collectd: false + create_grafana_annotations: false ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml diff --git a/conf/browbeat-keystone-minimal.yaml b/conf/browbeat-keystone-minimal.yaml index d1cd4c42d..8fae08e38 100644 --- a/conf/browbeat-keystone-minimal.yaml +++ b/conf/browbeat-keystone-minimal.yaml @@ -5,6 +5,7 @@ browbeat: rerun: 1 rerun_type: iteration start_stop_collectd: false + create_grafana_annotations: false ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml diff --git a/conf/browbeat-workloads.yml b/conf/browbeat-workloads.yml index 95dbfd66e..9eed23d7f 100644 --- a/conf/browbeat-workloads.yml +++ b/conf/browbeat-workloads.yml @@ -4,6 +4,7 @@ browbeat: rerun: 1 rerun_type: iteration start_stop_collectd: false + create_grafana_annotations: false ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml diff --git a/conf/ceilometer-minimal.yaml b/conf/ceilometer-minimal.yaml index dabf35eee..a24033c02 100644 --- a/conf/ceilometer-minimal.yaml +++ b/conf/ceilometer-minimal.yaml @@ -4,6 +4,7 @@ browbeat: rerun: 1 rerun_type: iteration start_stop_collectd: false + create_grafana_annotations: false ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml diff --git a/conf/ceilometer-persist-response-timing.yaml b/conf/ceilometer-persist-response-timing.yaml index 3492f5755..af3626721 100644 --- a/conf/ceilometer-persist-response-timing.yaml +++ b/conf/ceilometer-persist-response-timing.yaml @@ -7,6 +7,7 @@ browbeat: rerun: 10 rerun_type: complete start_stop_collectd: false + create_grafana_annotations: false ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml diff --git a/conf/gnocchi-minimal.yaml b/conf/gnocchi-minimal.yaml index 939569345..3fbd31560 100644 --- a/conf/gnocchi-minimal.yaml +++ b/conf/gnocchi-minimal.yaml @@ -5,6 +5,7 @@ browbeat: rerun: 1 rerun_type: iteration start_stop_collectd: false + create_grafana_annotations: false ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml diff --git a/conf/quickstart.yml b/conf/quickstart.yml index 5726234b2..96b034e04 100644 --- a/conf/quickstart.yml +++ b/conf/quickstart.yml @@ -4,6 +4,7 @@ browbeat: rerun: 1 rerun_type: iteration start_stop_collectd: false + create_grafana_annotations: false ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml diff --git a/conf/telemetry-nova-neutron-gnocchi-10k-1000-per-30m.yaml b/conf/telemetry-nova-neutron-gnocchi-10k-1000-per-30m.yaml index 4c6399f7a..4c38dd23c 100644 --- a/conf/telemetry-nova-neutron-gnocchi-10k-1000-per-30m.yaml +++ b/conf/telemetry-nova-neutron-gnocchi-10k-1000-per-30m.yaml @@ -11,6 +11,7 @@ browbeat: rerun: 10 rerun_type: complete start_stop_collectd: false + create_grafana_annotations: false ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml diff --git a/conf/telemetry-nova-neutron-gnocchi-1k-100-per-30m.yaml b/conf/telemetry-nova-neutron-gnocchi-1k-100-per-30m.yaml index 09e83d2d1..0aa006e3a 100644 --- a/conf/telemetry-nova-neutron-gnocchi-1k-100-per-30m.yaml +++ b/conf/telemetry-nova-neutron-gnocchi-1k-100-per-30m.yaml @@ -11,6 +11,7 @@ browbeat: rerun: 10 rerun_type: complete start_stop_collectd: false + create_grafana_annotations: false ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml diff --git a/conf/telemetry-nova-neutron-gnocchi-1k-500-per-30m.yaml b/conf/telemetry-nova-neutron-gnocchi-1k-500-per-30m.yaml index 948a091b7..5318b1a44 100644 --- a/conf/telemetry-nova-neutron-gnocchi-1k-500-per-30m.yaml +++ b/conf/telemetry-nova-neutron-gnocchi-1k-500-per-30m.yaml @@ -11,6 +11,7 @@ browbeat: rerun: 2 rerun_type: complete start_stop_collectd: false + create_grafana_annotations: false ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml diff --git a/conf/telemetry-nova-neutron-gnocchi-5k-500-per-30m.yaml b/conf/telemetry-nova-neutron-gnocchi-5k-500-per-30m.yaml index 8b4ee89cb..3fd89a1a5 100644 --- a/conf/telemetry-nova-neutron-gnocchi-5k-500-per-30m.yaml +++ b/conf/telemetry-nova-neutron-gnocchi-5k-500-per-30m.yaml @@ -11,6 +11,7 @@ browbeat: rerun: 10 rerun_type: complete start_stop_collectd: false + create_grafana_annotations: false ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml diff --git a/conf/telemetry-nova-persist-instances-with-network-fip.yaml b/conf/telemetry-nova-persist-instances-with-network-fip.yaml index f5709a9e3..14c603402 100644 --- a/conf/telemetry-nova-persist-instances-with-network-fip.yaml +++ b/conf/telemetry-nova-persist-instances-with-network-fip.yaml @@ -8,6 +8,7 @@ browbeat: rerun: 1 rerun_type: iteration start_stop_collectd: false + create_grafana_annotations: false ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml diff --git a/conf/telemetry-nova-persist-instances-with-network-volume-fip.yaml b/conf/telemetry-nova-persist-instances-with-network-volume-fip.yaml index 81ca23258..6e9417b0c 100644 --- a/conf/telemetry-nova-persist-instances-with-network-volume-fip.yaml +++ b/conf/telemetry-nova-persist-instances-with-network-volume-fip.yaml @@ -8,6 +8,7 @@ browbeat: rerun: 1 rerun_type: iteration start_stop_collectd: false + create_grafana_annotations: false ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml diff --git a/conf/telemetry-nova-persist-instances-with-network-volume.yaml b/conf/telemetry-nova-persist-instances-with-network-volume.yaml index b9b9c24a9..573cead24 100644 --- a/conf/telemetry-nova-persist-instances-with-network-volume.yaml +++ b/conf/telemetry-nova-persist-instances-with-network-volume.yaml @@ -7,6 +7,7 @@ browbeat: rerun: 1 rerun_type: iteration start_stop_collectd: false + create_grafana_annotations: false ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml diff --git a/conf/telemetry-nova-persist-instances-with-network.yaml b/conf/telemetry-nova-persist-instances-with-network.yaml index 86ca91980..b3cd08c06 100644 --- a/conf/telemetry-nova-persist-instances-with-network.yaml +++ b/conf/telemetry-nova-persist-instances-with-network.yaml @@ -9,6 +9,7 @@ browbeat: rerun: 1 rerun_type: iteration start_stop_collectd: false + create_grafana_annotations: false ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml diff --git a/conf/telemetry-nova-persist-instances-with-volume.yaml b/conf/telemetry-nova-persist-instances-with-volume.yaml index a91482679..ad702996c 100644 --- a/conf/telemetry-nova-persist-instances-with-volume.yaml +++ b/conf/telemetry-nova-persist-instances-with-volume.yaml @@ -7,6 +7,7 @@ browbeat: rerun: 1 rerun_type: iteration start_stop_collectd: false + create_grafana_annotations: false ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml diff --git a/conf/telemetry-nova-persist-instances.yaml b/conf/telemetry-nova-persist-instances.yaml index 135743f67..5ec444d4e 100644 --- a/conf/telemetry-nova-persist-instances.yaml +++ b/conf/telemetry-nova-persist-instances.yaml @@ -8,6 +8,7 @@ browbeat: rerun: 1 rerun_type: iteration start_stop_collectd: false + create_grafana_annotations: false ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml diff --git a/tests/data/invalid_browbeat.yml b/tests/data/invalid_browbeat.yml index 89d2656ea..cf2274e5a 100644 --- a/tests/data/invalid_browbeat.yml +++ b/tests/data/invalid_browbeat.yml @@ -5,6 +5,7 @@ browbeat: rerun_type: complete invalid_flag: invalid start_stop_collectd: false + create_grafana_annotations: false ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/site.yml diff --git a/tests/data/valid_browbeat.yml b/tests/data/valid_browbeat.yml index 8ff665dfd..c265ff32d 100644 --- a/tests/data/valid_browbeat.yml +++ b/tests/data/valid_browbeat.yml @@ -4,6 +4,7 @@ browbeat: rerun: 1 rerun_type: complete start_stop_collectd: false + create_grafana_annotations: false ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/site.yml