Grafana snapshot playbook.
+ Testing having ansible run the playbook + Added more panels to snapshoot + subprocess the snapshot to the background + create html linking undercloud/controllers and displaying pngs + Move graphs into test_name directory + Changed bg color, link color, added skipping computes by default + snapshot_compute exposed in browbeat-config.yaml Change-Id: I4e7cc55ab455e8a6cee82a72944d2be28c3fc772
This commit is contained in:
Alex Krzos
parent
e0a121b193
commit
f5b4c4d3f5
34
ansible/browbeat/roles/grafana-snapshot/tasks/main.yml
Normal file
34
ansible/browbeat/roles/grafana-snapshot/tasks/main.yml
Normal file
@ -0,0 +1,34 @@
|
||||
---
|
||||
#
|
||||
# Generate Snapshots
|
||||
#
|
||||
|
||||
- name: Generate General Snapshots
|
||||
shell: "mkdir -p /home/stack/browbeat/{{results_dir}}/{{item[0]}}/; curl -X GET -H 'Authorization: Bearer {{grafana_api_key}}' 'http://{{grafana_ip}}:{{grafana_port}}/render/dashboard-solo/db/openstack-general-system-performance?panelId={{item[1].panelId}}&from={{from}}&to={{to}}&var-Cloud={{var_cloud}}&var-Node={{item[0]}}_localdomain&var-Interface=interface-test&var-Disk=disk-sda&width=1200' > /home/stack/browbeat/{{results_dir}}/{{item[0]}}/{{item[0]}}-{{item[1].name}}.png"
|
||||
with_nested:
|
||||
- "{{ hosts_in_group }}"
|
||||
- "{{ general_panels }}"
|
||||
|
||||
- name: Generate Disk Snapshots
|
||||
shell: "mkdir -p /home/stack/browbeat/{{results_dir}}/{{item[0]}}/;curl -X GET -H 'Authorization: Bearer {{grafana_api_key}}' 'http://{{grafana_ip}}:{{grafana_port}}/render/dashboard-solo/db/openstack-general-system-performance?panelId={{item[2].panelId}}&from={{from}}&to={{to}}&var-Cloud={{var_cloud}}&var-Node={{item[0]}}_localdomain&var-Interface=interface-test&var-Disk=disk-{{item[1]}}&width=1200' > /home/stack/browbeat/{{results_dir}}/{{item[0]}}/{{item[0]}}-{{item[2].name}}-{{item[1]}}.png"
|
||||
with_nested:
|
||||
- "{{ hosts_in_group }}"
|
||||
- "{{ disks_in_group }}"
|
||||
- "{{ disk_panels }}"
|
||||
|
||||
- name: Generate Interface Snapshots
|
||||
shell: "mkdir -p /home/stack/browbeat/{{results_dir}}/{{item[0]}}/;curl -X GET -H 'Authorization: Bearer {{grafana_api_key}}' 'http://{{grafana_ip}}:{{grafana_port}}/render/dashboard-solo/db/openstack-general-system-performance?panelId={{item[2].panelId}}&from={{from}}&to={{to}}&var-Cloud={{var_cloud}}&var-Node={{item[0]}}_localdomain&var-Interface=interface-{{item[1]}}&var-Disk=disk-sda&width=1200' > /home/stack/browbeat/{{results_dir}}/{{item[0]}}/{{item[0]}}-{{item[2].name}}-{{item[1]}}.png"
|
||||
with_nested:
|
||||
- "{{ hosts_in_group }}"
|
||||
- "{{ interfaces_in_group }}"
|
||||
- "{{ interface_panels }}"
|
||||
|
||||
- name: Generate index.html per host
|
||||
template:
|
||||
src=index.html.j2
|
||||
dest=/home/stack/browbeat/{{results_dir}}/{{item}}/index.html
|
||||
owner=stack
|
||||
group=stack
|
||||
mode=0644
|
||||
with_items:
|
||||
- "{{groups[host_type]}}"
|
||||
@ -0,0 +1,60 @@
|
||||
<html>
|
||||
<head>
|
||||
<title>{{item}}</title>
|
||||
<style type="text/css">
|
||||
a:link { color: #c00 } /* unvisited */
|
||||
a:visited { color: #0c0 } /* visited */
|
||||
a:hover { color: #00c } /* hovers */
|
||||
a:active { color: #ccc } /* active */
|
||||
</style>
|
||||
</head>
|
||||
<body style="background-color: rgb(22, 22, 22)">
|
||||
<div style="color: rgb(255,255,255)">
|
||||
<b>{{item}} - System Performance Snapshot</b>
|
||||
<br>
|
||||
Undercloud:
|
||||
{% for host in groups['undercloud'] %}
|
||||
<a href="../{{host}}">{{host}}</a>
|
||||
{% endfor %}
|
||||
<br>
|
||||
Controllers:
|
||||
{% for host in groups['controller'] %}
|
||||
<a href="../{{host}}">{{host}}</a>
|
||||
{% endfor %}
|
||||
{% if snapshot_compute %}
|
||||
<br>
|
||||
Computes:
|
||||
{% for host in groups['compute'] %}
|
||||
<a href="../{{host}}">{{host}}</a>
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
</div>
|
||||
<br>
|
||||
<a href="#disks">Disks</a> :
|
||||
{% for disk in disks_in_group %}
|
||||
<a href="#disk-{{disk}}">{{disk}}</a>
|
||||
{% endfor %}
|
||||
<br><br>
|
||||
<a href="#interfaces">Interfaces</a> :
|
||||
{% for interface in interfaces_in_group %}
|
||||
<a href="#interface-{{interface}}">{{interface}}</a>
|
||||
{% endfor %}
|
||||
<br><br>
|
||||
{% for panel in general_panels %}
|
||||
<img src="{{item}}-{{panel.name}}.png"/><br>
|
||||
{% endfor %}
|
||||
<div id="disks">
|
||||
{% for disk in disks_in_group %}
|
||||
{% for panel in disk_panels %}
|
||||
<div id="disk-{{disk}}">
|
||||
<img src="{{item}}-{{panel.name}}-{{disk}}.png"/><br>
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
<div id="interfaces">
|
||||
{% for interface in interfaces_in_group %}
|
||||
{% for panel in interface_panels %}
|
||||
<div id="interface-{{interface}}">
|
||||
<img src="{{item}}-{{panel.name}}-{{interface}}.png"/><br>
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
</html>
|
||||
95
ansible/browbeat/roles/grafana-snapshot/vars/main.yml
Normal file
95
ansible/browbeat/roles/grafana-snapshot/vars/main.yml
Normal file
@ -0,0 +1,95 @@
|
||||
---
|
||||
#
|
||||
# Grafana Snapshot vars
|
||||
#
|
||||
|
||||
# Task prepends "disk-" prefix for you
|
||||
disks:
|
||||
undercloud:
|
||||
- vda
|
||||
controller:
|
||||
- sda
|
||||
- sda2
|
||||
compute:
|
||||
- sda
|
||||
- sda2
|
||||
|
||||
disk_panels:
|
||||
- panelId: 6
|
||||
name: disk-iops
|
||||
- panelId: 7
|
||||
name: disk-throughput
|
||||
|
||||
# Task prepends "interface-" prefix for you
|
||||
interfaces:
|
||||
undercloud:
|
||||
- br-ctlplane
|
||||
# - br-int
|
||||
- ens3
|
||||
- ens7
|
||||
- lo
|
||||
# - ovs-system
|
||||
controller:
|
||||
- br-ex
|
||||
- br-int
|
||||
- br-tun
|
||||
- br-vlan
|
||||
- enp3s0f0
|
||||
- enp3s0f1
|
||||
- enp4s0f0
|
||||
- enp4s0f1
|
||||
# - ens1f0
|
||||
# - ens1f1
|
||||
- lo
|
||||
# - ovs-system
|
||||
- vlan201
|
||||
- vlan202
|
||||
- vlan203
|
||||
- vlan204
|
||||
compute:
|
||||
- br-ex
|
||||
- br-int
|
||||
- br-tun
|
||||
- br-vlan
|
||||
- enp3s0f0
|
||||
- enp3s0f1
|
||||
- enp4s0f0
|
||||
- enp4s0f1
|
||||
# - ens1f0
|
||||
# - ens1f1
|
||||
- lo
|
||||
# - ovs-system
|
||||
- vlan201
|
||||
- vlan202
|
||||
- vlan203
|
||||
- vlan204
|
||||
|
||||
interface_panels:
|
||||
- panelId: 9
|
||||
name: network-pps
|
||||
- panelId: 10
|
||||
name: network-throughput
|
||||
|
||||
general_panels:
|
||||
- panelId: 1
|
||||
name: all-cpu
|
||||
- panelId: 2
|
||||
name: memory-bytes
|
||||
- panelId: 3
|
||||
name: memory-percentage
|
||||
- panelId: 4
|
||||
name: swap-usage
|
||||
- panelId: 11
|
||||
name: conntrack
|
||||
- panelId: 15
|
||||
name: numa
|
||||
- panelId: 16
|
||||
name: irq
|
||||
- panelId: 17
|
||||
name: interrupts
|
||||
- panelId: 18
|
||||
name: load-uptime
|
||||
- panelId: 19
|
||||
name: processes
|
||||
- panelId: 92
|
||||
name: tail-errors
|
||||
42
ansible/browbeat/snapshot-general-performance-dashboard.yml
Normal file
42
ansible/browbeat/snapshot-general-performance-dashboard.yml
Normal file
@ -0,0 +1,42 @@
|
||||
---
|
||||
#
|
||||
# Snapshot Dashboard
|
||||
#
|
||||
# Example Usage:
|
||||
# ansible-playbook -i hosts browbeat/snapshot-general-performance-dashboard.yml -e "grafana_ip=1.1.1.1 grafana_port=3000 grafana_api_key=<apikey> from=1455649200000 to=1455656400000 results_dir=results/ var_cloud=openstack"
|
||||
#
|
||||
# Append snapshot_compute=true to run snapshots against computes.
|
||||
#
|
||||
|
||||
- hosts: undercloud
|
||||
gather_facts: false
|
||||
remote_user: stack
|
||||
vars:
|
||||
host_type: undercloud
|
||||
hosts_in_group: groups['undercloud']
|
||||
disks_in_group: "{{disks['undercloud']}}"
|
||||
interfaces_in_group: "{{interfaces['undercloud']}}"
|
||||
roles:
|
||||
- grafana-snapshot
|
||||
|
||||
- hosts: undercloud
|
||||
gather_facts: false
|
||||
remote_user: stack
|
||||
vars:
|
||||
host_type: controller
|
||||
hosts_in_group: groups['controller']
|
||||
disks_in_group: "{{disks['controller']}}"
|
||||
interfaces_in_group: "{{interfaces['controller']}}"
|
||||
roles:
|
||||
- grafana-snapshot
|
||||
|
||||
- hosts: undercloud
|
||||
gather_facts: false
|
||||
remote_user: stack
|
||||
vars:
|
||||
host_type: compute
|
||||
hosts_in_group: groups['compute']
|
||||
disks_in_group: "{{disks['compute']}}"
|
||||
interfaces_in_group: "{{interfaces['compute']}}"
|
||||
roles:
|
||||
- { role: grafana-snapshot, when: snapshot_compute is defined }
|
||||
@ -15,12 +15,18 @@ ansible:
|
||||
neutron_l3: ansible/browbeat/adjustment-l3.yml
|
||||
nova_db: ansible/browbeat/adjustment-db.yml
|
||||
workers: ansible/browbeat/adjustment-workers.yml
|
||||
grafana_snapshot: ansible/browbeat/snapshot-general-performance-dashboard.yml
|
||||
grafana:
|
||||
enabled: true
|
||||
cloud_name: openstack
|
||||
grafana_ip: 1.1.1.1
|
||||
grafana_port: 3000
|
||||
dashboards:
|
||||
- openstack-general-system-performance
|
||||
url: http://grafana-host.example.com:3000/dashboard/db/
|
||||
snapshot:
|
||||
enabled: true
|
||||
grafana_api_key: (Your API Key Here)
|
||||
snapshot_compute: false
|
||||
rally:
|
||||
sleep_before: 5
|
||||
sleep_after: 5
|
||||
|
||||
@ -15,12 +15,18 @@ ansible:
|
||||
neutron_l3: ansible/browbeat/adjustment-l3.yml
|
||||
nova_db: ansible/browbeat/adjustment-db.yml
|
||||
workers: ansible/browbeat/adjustment-workers.yml
|
||||
grafana_snapshot: ansible/browbeat/snapshot-general-performance-dashboard.yml
|
||||
grafana:
|
||||
enabled: true
|
||||
cloud_name: openstack
|
||||
grafana_ip: 1.1.1.1
|
||||
grafana_port: 3000
|
||||
dashboards:
|
||||
- openstack-general-system-performance
|
||||
url: http://grafana-host.example.com:3000/dashboard/db/
|
||||
snapshot:
|
||||
enabled: true
|
||||
grafana_api_key: (Your API Key Here)
|
||||
snapshot_compute: false
|
||||
rally:
|
||||
sleep_before: 5
|
||||
sleep_after: 5
|
||||
|
||||
24
lib/Rally.py
24
lib/Rally.py
@ -6,6 +6,7 @@ import glob
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
|
||||
@ -40,12 +41,33 @@ class Rally:
|
||||
to_ts = int(time.time() * 1000)
|
||||
|
||||
if 'grafana' in self.config and self.config['grafana']['enabled']:
|
||||
url = self.config['grafana']['url']
|
||||
grafana_ip = self.config['grafana']['grafana_ip']
|
||||
grafana_port = self.config['grafana']['grafana_port']
|
||||
url = 'http://{}:{}/dashboard/db/'.format(grafana_ip, grafana_port)
|
||||
cloud_name = self.config['grafana']['cloud_name']
|
||||
for dashboard in self.config['grafana']['dashboards']:
|
||||
full_url = '{}{}?from={}&to={}&var-Cloud={}'.format(url, dashboard, from_ts, to_ts,
|
||||
cloud_name)
|
||||
self.logger.info('{} - Grafana URL: {}'.format(test_name, full_url))
|
||||
if self.config['grafana']['snapshot']['enabled']:
|
||||
hosts_file = self.config['ansible']['hosts']
|
||||
playbook = self.config['ansible']['grafana_snapshot']
|
||||
extra_vars = 'grafana_ip={} '.format(grafana_ip)
|
||||
extra_vars += 'grafana_port={} '.format(grafana_port)
|
||||
extra_vars += 'grafana_api_key={} '.format(self.config['grafana']['snapshot']['grafana_api_key'])
|
||||
extra_vars += 'from={} '.format(from_ts)
|
||||
extra_vars += 'to={} '.format(to_ts)
|
||||
extra_vars += 'results_dir={}/{} '.format(result_dir, test_name)
|
||||
extra_vars += 'var_cloud={} '.format(cloud_name)
|
||||
if self.config['grafana']['snapshot']['snapshot_compute']:
|
||||
extra_vars += 'snapshot_compute=true '
|
||||
snapshot_cmd = 'ansible-playbook -i {} {} -e "{}"'.format(hosts_file, playbook,
|
||||
extra_vars)
|
||||
subprocess_cmd = ['ansible-playbook', '-i', hosts_file, playbook, '-e',
|
||||
'{}'.format(extra_vars)]
|
||||
self.logger.info('Snapshot command: {}'.format(snapshot_cmd))
|
||||
snapshot_log = open('{}/snapshot.log'.format(result_dir), 'a+')
|
||||
subprocess.Popen(subprocess_cmd, stdout=snapshot_log, stderr=subprocess.STDOUT)
|
||||
|
||||
def workload_logger(self, result_dir):
|
||||
base = result_dir.split('/')
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user