Grafana snapshot playbook.
+ Testing having ansible run the playbook + Added more panels to snapshoot + subprocess the snapshot to the background + create html linking undercloud/controllers and displaying pngs + Move graphs into test_name directory + Changed bg color, link color, added skipping computes by default + snapshot_compute exposed in browbeat-config.yaml Change-Id: I4e7cc55ab455e8a6cee82a72944d2be28c3fc772
This commit is contained in:
Alex Krzos
parent
e0a121b193
commit
f5b4c4d3f5
34
ansible/browbeat/roles/grafana-snapshot/tasks/main.yml
Normal file
34
ansible/browbeat/roles/grafana-snapshot/tasks/main.yml
Normal file
@ -0,0 +1,34 @@
|
|||||||
|
---
|
||||||
|
#
|
||||||
|
# Generate Snapshots
|
||||||
|
#
|
||||||
|
|
||||||
|
- name: Generate General Snapshots
|
||||||
|
shell: "mkdir -p /home/stack/browbeat/{{results_dir}}/{{item[0]}}/; curl -X GET -H 'Authorization: Bearer {{grafana_api_key}}' 'http://{{grafana_ip}}:{{grafana_port}}/render/dashboard-solo/db/openstack-general-system-performance?panelId={{item[1].panelId}}&from={{from}}&to={{to}}&var-Cloud={{var_cloud}}&var-Node={{item[0]}}_localdomain&var-Interface=interface-test&var-Disk=disk-sda&width=1200' > /home/stack/browbeat/{{results_dir}}/{{item[0]}}/{{item[0]}}-{{item[1].name}}.png"
|
||||||
|
with_nested:
|
||||||
|
- "{{ hosts_in_group }}"
|
||||||
|
- "{{ general_panels }}"
|
||||||
|
|
||||||
|
- name: Generate Disk Snapshots
|
||||||
|
shell: "mkdir -p /home/stack/browbeat/{{results_dir}}/{{item[0]}}/;curl -X GET -H 'Authorization: Bearer {{grafana_api_key}}' 'http://{{grafana_ip}}:{{grafana_port}}/render/dashboard-solo/db/openstack-general-system-performance?panelId={{item[2].panelId}}&from={{from}}&to={{to}}&var-Cloud={{var_cloud}}&var-Node={{item[0]}}_localdomain&var-Interface=interface-test&var-Disk=disk-{{item[1]}}&width=1200' > /home/stack/browbeat/{{results_dir}}/{{item[0]}}/{{item[0]}}-{{item[2].name}}-{{item[1]}}.png"
|
||||||
|
with_nested:
|
||||||
|
- "{{ hosts_in_group }}"
|
||||||
|
- "{{ disks_in_group }}"
|
||||||
|
- "{{ disk_panels }}"
|
||||||
|
|
||||||
|
- name: Generate Interface Snapshots
|
||||||
|
shell: "mkdir -p /home/stack/browbeat/{{results_dir}}/{{item[0]}}/;curl -X GET -H 'Authorization: Bearer {{grafana_api_key}}' 'http://{{grafana_ip}}:{{grafana_port}}/render/dashboard-solo/db/openstack-general-system-performance?panelId={{item[2].panelId}}&from={{from}}&to={{to}}&var-Cloud={{var_cloud}}&var-Node={{item[0]}}_localdomain&var-Interface=interface-{{item[1]}}&var-Disk=disk-sda&width=1200' > /home/stack/browbeat/{{results_dir}}/{{item[0]}}/{{item[0]}}-{{item[2].name}}-{{item[1]}}.png"
|
||||||
|
with_nested:
|
||||||
|
- "{{ hosts_in_group }}"
|
||||||
|
- "{{ interfaces_in_group }}"
|
||||||
|
- "{{ interface_panels }}"
|
||||||
|
|
||||||
|
- name: Generate index.html per host
|
||||||
|
template:
|
||||||
|
src=index.html.j2
|
||||||
|
dest=/home/stack/browbeat/{{results_dir}}/{{item}}/index.html
|
||||||
|
owner=stack
|
||||||
|
group=stack
|
||||||
|
mode=0644
|
||||||
|
with_items:
|
||||||
|
- "{{groups[host_type]}}"
|
||||||
@ -0,0 +1,60 @@
|
|||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<title>{{item}}</title>
|
||||||
|
<style type="text/css">
|
||||||
|
a:link { color: #c00 } /* unvisited */
|
||||||
|
a:visited { color: #0c0 } /* visited */
|
||||||
|
a:hover { color: #00c } /* hovers */
|
||||||
|
a:active { color: #ccc } /* active */
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body style="background-color: rgb(22, 22, 22)">
|
||||||
|
<div style="color: rgb(255,255,255)">
|
||||||
|
<b>{{item}} - System Performance Snapshot</b>
|
||||||
|
<br>
|
||||||
|
Undercloud:
|
||||||
|
{% for host in groups['undercloud'] %}
|
||||||
|
<a href="../{{host}}">{{host}}</a>
|
||||||
|
{% endfor %}
|
||||||
|
<br>
|
||||||
|
Controllers:
|
||||||
|
{% for host in groups['controller'] %}
|
||||||
|
<a href="../{{host}}">{{host}}</a>
|
||||||
|
{% endfor %}
|
||||||
|
{% if snapshot_compute %}
|
||||||
|
<br>
|
||||||
|
Computes:
|
||||||
|
{% for host in groups['compute'] %}
|
||||||
|
<a href="../{{host}}">{{host}}</a>
|
||||||
|
{% endfor %}
|
||||||
|
{% endif %}
|
||||||
|
</div>
|
||||||
|
<br>
|
||||||
|
<a href="#disks">Disks</a> :
|
||||||
|
{% for disk in disks_in_group %}
|
||||||
|
<a href="#disk-{{disk}}">{{disk}}</a>
|
||||||
|
{% endfor %}
|
||||||
|
<br><br>
|
||||||
|
<a href="#interfaces">Interfaces</a> :
|
||||||
|
{% for interface in interfaces_in_group %}
|
||||||
|
<a href="#interface-{{interface}}">{{interface}}</a>
|
||||||
|
{% endfor %}
|
||||||
|
<br><br>
|
||||||
|
{% for panel in general_panels %}
|
||||||
|
<img src="{{item}}-{{panel.name}}.png"/><br>
|
||||||
|
{% endfor %}
|
||||||
|
<div id="disks">
|
||||||
|
{% for disk in disks_in_group %}
|
||||||
|
{% for panel in disk_panels %}
|
||||||
|
<div id="disk-{{disk}}">
|
||||||
|
<img src="{{item}}-{{panel.name}}-{{disk}}.png"/><br>
|
||||||
|
{% endfor %}
|
||||||
|
{% endfor %}
|
||||||
|
<div id="interfaces">
|
||||||
|
{% for interface in interfaces_in_group %}
|
||||||
|
{% for panel in interface_panels %}
|
||||||
|
<div id="interface-{{interface}}">
|
||||||
|
<img src="{{item}}-{{panel.name}}-{{interface}}.png"/><br>
|
||||||
|
{% endfor %}
|
||||||
|
{% endfor %}
|
||||||
|
</html>
|
||||||
95
ansible/browbeat/roles/grafana-snapshot/vars/main.yml
Normal file
95
ansible/browbeat/roles/grafana-snapshot/vars/main.yml
Normal file
@ -0,0 +1,95 @@
|
|||||||
|
---
|
||||||
|
#
|
||||||
|
# Grafana Snapshot vars
|
||||||
|
#
|
||||||
|
|
||||||
|
# Task prepends "disk-" prefix for you
|
||||||
|
disks:
|
||||||
|
undercloud:
|
||||||
|
- vda
|
||||||
|
controller:
|
||||||
|
- sda
|
||||||
|
- sda2
|
||||||
|
compute:
|
||||||
|
- sda
|
||||||
|
- sda2
|
||||||
|
|
||||||
|
disk_panels:
|
||||||
|
- panelId: 6
|
||||||
|
name: disk-iops
|
||||||
|
- panelId: 7
|
||||||
|
name: disk-throughput
|
||||||
|
|
||||||
|
# Task prepends "interface-" prefix for you
|
||||||
|
interfaces:
|
||||||
|
undercloud:
|
||||||
|
- br-ctlplane
|
||||||
|
# - br-int
|
||||||
|
- ens3
|
||||||
|
- ens7
|
||||||
|
- lo
|
||||||
|
# - ovs-system
|
||||||
|
controller:
|
||||||
|
- br-ex
|
||||||
|
- br-int
|
||||||
|
- br-tun
|
||||||
|
- br-vlan
|
||||||
|
- enp3s0f0
|
||||||
|
- enp3s0f1
|
||||||
|
- enp4s0f0
|
||||||
|
- enp4s0f1
|
||||||
|
# - ens1f0
|
||||||
|
# - ens1f1
|
||||||
|
- lo
|
||||||
|
# - ovs-system
|
||||||
|
- vlan201
|
||||||
|
- vlan202
|
||||||
|
- vlan203
|
||||||
|
- vlan204
|
||||||
|
compute:
|
||||||
|
- br-ex
|
||||||
|
- br-int
|
||||||
|
- br-tun
|
||||||
|
- br-vlan
|
||||||
|
- enp3s0f0
|
||||||
|
- enp3s0f1
|
||||||
|
- enp4s0f0
|
||||||
|
- enp4s0f1
|
||||||
|
# - ens1f0
|
||||||
|
# - ens1f1
|
||||||
|
- lo
|
||||||
|
# - ovs-system
|
||||||
|
- vlan201
|
||||||
|
- vlan202
|
||||||
|
- vlan203
|
||||||
|
- vlan204
|
||||||
|
|
||||||
|
interface_panels:
|
||||||
|
- panelId: 9
|
||||||
|
name: network-pps
|
||||||
|
- panelId: 10
|
||||||
|
name: network-throughput
|
||||||
|
|
||||||
|
general_panels:
|
||||||
|
- panelId: 1
|
||||||
|
name: all-cpu
|
||||||
|
- panelId: 2
|
||||||
|
name: memory-bytes
|
||||||
|
- panelId: 3
|
||||||
|
name: memory-percentage
|
||||||
|
- panelId: 4
|
||||||
|
name: swap-usage
|
||||||
|
- panelId: 11
|
||||||
|
name: conntrack
|
||||||
|
- panelId: 15
|
||||||
|
name: numa
|
||||||
|
- panelId: 16
|
||||||
|
name: irq
|
||||||
|
- panelId: 17
|
||||||
|
name: interrupts
|
||||||
|
- panelId: 18
|
||||||
|
name: load-uptime
|
||||||
|
- panelId: 19
|
||||||
|
name: processes
|
||||||
|
- panelId: 92
|
||||||
|
name: tail-errors
|
||||||
42
ansible/browbeat/snapshot-general-performance-dashboard.yml
Normal file
42
ansible/browbeat/snapshot-general-performance-dashboard.yml
Normal file
@ -0,0 +1,42 @@
|
|||||||
|
---
|
||||||
|
#
|
||||||
|
# Snapshot Dashboard
|
||||||
|
#
|
||||||
|
# Example Usage:
|
||||||
|
# ansible-playbook -i hosts browbeat/snapshot-general-performance-dashboard.yml -e "grafana_ip=1.1.1.1 grafana_port=3000 grafana_api_key=<apikey> from=1455649200000 to=1455656400000 results_dir=results/ var_cloud=openstack"
|
||||||
|
#
|
||||||
|
# Append snapshot_compute=true to run snapshots against computes.
|
||||||
|
#
|
||||||
|
|
||||||
|
- hosts: undercloud
|
||||||
|
gather_facts: false
|
||||||
|
remote_user: stack
|
||||||
|
vars:
|
||||||
|
host_type: undercloud
|
||||||
|
hosts_in_group: groups['undercloud']
|
||||||
|
disks_in_group: "{{disks['undercloud']}}"
|
||||||
|
interfaces_in_group: "{{interfaces['undercloud']}}"
|
||||||
|
roles:
|
||||||
|
- grafana-snapshot
|
||||||
|
|
||||||
|
- hosts: undercloud
|
||||||
|
gather_facts: false
|
||||||
|
remote_user: stack
|
||||||
|
vars:
|
||||||
|
host_type: controller
|
||||||
|
hosts_in_group: groups['controller']
|
||||||
|
disks_in_group: "{{disks['controller']}}"
|
||||||
|
interfaces_in_group: "{{interfaces['controller']}}"
|
||||||
|
roles:
|
||||||
|
- grafana-snapshot
|
||||||
|
|
||||||
|
- hosts: undercloud
|
||||||
|
gather_facts: false
|
||||||
|
remote_user: stack
|
||||||
|
vars:
|
||||||
|
host_type: compute
|
||||||
|
hosts_in_group: groups['compute']
|
||||||
|
disks_in_group: "{{disks['compute']}}"
|
||||||
|
interfaces_in_group: "{{interfaces['compute']}}"
|
||||||
|
roles:
|
||||||
|
- { role: grafana-snapshot, when: snapshot_compute is defined }
|
||||||
@ -15,12 +15,18 @@ ansible:
|
|||||||
neutron_l3: ansible/browbeat/adjustment-l3.yml
|
neutron_l3: ansible/browbeat/adjustment-l3.yml
|
||||||
nova_db: ansible/browbeat/adjustment-db.yml
|
nova_db: ansible/browbeat/adjustment-db.yml
|
||||||
workers: ansible/browbeat/adjustment-workers.yml
|
workers: ansible/browbeat/adjustment-workers.yml
|
||||||
|
grafana_snapshot: ansible/browbeat/snapshot-general-performance-dashboard.yml
|
||||||
grafana:
|
grafana:
|
||||||
enabled: true
|
enabled: true
|
||||||
cloud_name: openstack
|
cloud_name: openstack
|
||||||
|
grafana_ip: 1.1.1.1
|
||||||
|
grafana_port: 3000
|
||||||
dashboards:
|
dashboards:
|
||||||
- openstack-general-system-performance
|
- openstack-general-system-performance
|
||||||
url: http://grafana-host.example.com:3000/dashboard/db/
|
snapshot:
|
||||||
|
enabled: true
|
||||||
|
grafana_api_key: (Your API Key Here)
|
||||||
|
snapshot_compute: false
|
||||||
rally:
|
rally:
|
||||||
sleep_before: 5
|
sleep_before: 5
|
||||||
sleep_after: 5
|
sleep_after: 5
|
||||||
|
|||||||
@ -15,12 +15,18 @@ ansible:
|
|||||||
neutron_l3: ansible/browbeat/adjustment-l3.yml
|
neutron_l3: ansible/browbeat/adjustment-l3.yml
|
||||||
nova_db: ansible/browbeat/adjustment-db.yml
|
nova_db: ansible/browbeat/adjustment-db.yml
|
||||||
workers: ansible/browbeat/adjustment-workers.yml
|
workers: ansible/browbeat/adjustment-workers.yml
|
||||||
|
grafana_snapshot: ansible/browbeat/snapshot-general-performance-dashboard.yml
|
||||||
grafana:
|
grafana:
|
||||||
enabled: true
|
enabled: true
|
||||||
cloud_name: openstack
|
cloud_name: openstack
|
||||||
|
grafana_ip: 1.1.1.1
|
||||||
|
grafana_port: 3000
|
||||||
dashboards:
|
dashboards:
|
||||||
- openstack-general-system-performance
|
- openstack-general-system-performance
|
||||||
url: http://grafana-host.example.com:3000/dashboard/db/
|
snapshot:
|
||||||
|
enabled: true
|
||||||
|
grafana_api_key: (Your API Key Here)
|
||||||
|
snapshot_compute: false
|
||||||
rally:
|
rally:
|
||||||
sleep_before: 5
|
sleep_before: 5
|
||||||
sleep_after: 5
|
sleep_after: 5
|
||||||
|
|||||||
24
lib/Rally.py
24
lib/Rally.py
@ -6,6 +6,7 @@ import glob
|
|||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
|
import subprocess
|
||||||
import time
|
import time
|
||||||
|
|
||||||
|
|
||||||
@ -40,12 +41,33 @@ class Rally:
|
|||||||
to_ts = int(time.time() * 1000)
|
to_ts = int(time.time() * 1000)
|
||||||
|
|
||||||
if 'grafana' in self.config and self.config['grafana']['enabled']:
|
if 'grafana' in self.config and self.config['grafana']['enabled']:
|
||||||
url = self.config['grafana']['url']
|
grafana_ip = self.config['grafana']['grafana_ip']
|
||||||
|
grafana_port = self.config['grafana']['grafana_port']
|
||||||
|
url = 'http://{}:{}/dashboard/db/'.format(grafana_ip, grafana_port)
|
||||||
cloud_name = self.config['grafana']['cloud_name']
|
cloud_name = self.config['grafana']['cloud_name']
|
||||||
for dashboard in self.config['grafana']['dashboards']:
|
for dashboard in self.config['grafana']['dashboards']:
|
||||||
full_url = '{}{}?from={}&to={}&var-Cloud={}'.format(url, dashboard, from_ts, to_ts,
|
full_url = '{}{}?from={}&to={}&var-Cloud={}'.format(url, dashboard, from_ts, to_ts,
|
||||||
cloud_name)
|
cloud_name)
|
||||||
self.logger.info('{} - Grafana URL: {}'.format(test_name, full_url))
|
self.logger.info('{} - Grafana URL: {}'.format(test_name, full_url))
|
||||||
|
if self.config['grafana']['snapshot']['enabled']:
|
||||||
|
hosts_file = self.config['ansible']['hosts']
|
||||||
|
playbook = self.config['ansible']['grafana_snapshot']
|
||||||
|
extra_vars = 'grafana_ip={} '.format(grafana_ip)
|
||||||
|
extra_vars += 'grafana_port={} '.format(grafana_port)
|
||||||
|
extra_vars += 'grafana_api_key={} '.format(self.config['grafana']['snapshot']['grafana_api_key'])
|
||||||
|
extra_vars += 'from={} '.format(from_ts)
|
||||||
|
extra_vars += 'to={} '.format(to_ts)
|
||||||
|
extra_vars += 'results_dir={}/{} '.format(result_dir, test_name)
|
||||||
|
extra_vars += 'var_cloud={} '.format(cloud_name)
|
||||||
|
if self.config['grafana']['snapshot']['snapshot_compute']:
|
||||||
|
extra_vars += 'snapshot_compute=true '
|
||||||
|
snapshot_cmd = 'ansible-playbook -i {} {} -e "{}"'.format(hosts_file, playbook,
|
||||||
|
extra_vars)
|
||||||
|
subprocess_cmd = ['ansible-playbook', '-i', hosts_file, playbook, '-e',
|
||||||
|
'{}'.format(extra_vars)]
|
||||||
|
self.logger.info('Snapshot command: {}'.format(snapshot_cmd))
|
||||||
|
snapshot_log = open('{}/snapshot.log'.format(result_dir), 'a+')
|
||||||
|
subprocess.Popen(subprocess_cmd, stdout=snapshot_log, stderr=subprocess.STDOUT)
|
||||||
|
|
||||||
def workload_logger(self, result_dir):
|
def workload_logger(self, result_dir):
|
||||||
base = result_dir.split('/')
|
base = result_dir.split('/')
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user