x/browbeat
Code Issues Proposed changes

Adding lib/Grafana

Browse Source 
Refactoring some code to make use of the library

+ Splitting into methods
+ Making methods independent with akrzos suggestions

Change-Id: I7a18d4c506a5dcd28d96872226e5001d4603cf48
This commit is contained in:
Sindhur Malleni 2016-03-13 16:24:45 -04:00 committed by Alex Krzos
parent a7964939df
commit 8ec0d9b3a5
6 changed files with 80 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
browbeat-complete.yaml
View File

@ -2,7 +2,7 @@
browbeat:
results : results/
sudo: true
connmon: true
connmon: false
rerun: 3
ansible:
hosts: ansible/hosts
@ -189,6 +189,8 @@ shaker:
port: 5555
flavor: m1.small
join_timeout: 600
sleep_before: 5
sleep_after: 5
venv: /home/stack/shaker-venv
scenarios:
l2-4-1:

6
browbeat-config.yaml
View File

@ -2,7 +2,7 @@
browbeat:
results : results/
sudo: true
connmon: true
connmon: false
rerun: 1
ansible:
hosts: ansible/hosts
@ -47,10 +47,12 @@ perfkit:
data_disk_size: 4
shaker:
enabled: true
server: (Address of machine running browbeat)
server: (Address of machine running browbeat-Undercloud)
port: 5555
flavor: m1.small
join_timeout: 600
sleep_before: 5
sleep_after: 5
venv: /home/stack/shaker-venv
scenarios:
l2:

49
lib/Grafana.py Normal file
View File

@ -0,0 +1,49 @@
import logging
import subprocess
class Grafana:
def __init__(self, config):
self.logger = logging.getLogger('browbeat.Grafana')
self.config = config
self.cloud_name = self.config['grafana']['cloud_name']
self.hosts_file = self.config['ansible']['hosts']
self.grafana_ip = self.config['grafana']['grafana_ip']
self.grafana_port = self.config['grafana']['grafana_port']
self.playbook = self.config['ansible']['grafana_snapshot']
def get_extra_vars(self, from_ts, to_ts, result_dir, test_name):
extra_vars = 'grafana_ip={} '.format(self.config['grafana']['grafana_ip'])
extra_vars += 'grafana_port={} '.format(self.config['grafana']['grafana_port'])
extra_vars += 'grafana_api_key={} '.format(self.config['grafana']['snapshot']['grafana_api_key'])
extra_vars += 'from={} '.format(from_ts)
extra_vars += 'to={} '.format(to_ts)
extra_vars += 'results_dir={}/{} '.format(result_dir, test_name)
extra_vars += 'var_cloud={} '.format(self.cloud_name)
if self.config['grafana']['snapshot']['snapshot_compute']:
extra_vars += 'snapshot_compute=true '
return extra_vars
def print_dashboard_url(self, from_ts, to_ts, test_name):
if 'grafana' in self.config and self.config['grafana']['enabled']:
url = 'http://{}:{}/dashboard/db/'.format(self.grafana_ip, self.grafana_port)
for dashboard in self.config['grafana']['dashboards']:
full_url = '{}{}?from={}&to={}&var-Cloud={}'.format(url, dashboard, from_ts, to_ts,
self.cloud_name)
self.logger.info('{} - Grafana URL: {}'.format(test_name, full_url))
def log_snapshot_playbook_cmd(self, from_ts, to_ts, result_dir, test_name):
if 'grafana' in self.config and self.config['grafana']['enabled']:
extra_vars = self.get_extra_vars(from_ts, to_ts, result_dir, test_name)
snapshot_cmd = 'ansible-playbook -i {} {} -e "{}"'.format(self.hosts_file, self.playbook,
extra_vars)
self.logger.info('Snapshot command: {}'.format(snapshot_cmd))
def run_playbook(self, from_ts, to_ts, result_dir, test_name):
if 'grafana' in self.config and self.config['grafana']['enabled']:
extra_vars = self.get_extra_vars(from_ts, to_ts, result_dir, test_name)
subprocess_cmd = ['ansible-playbook', '-i', self.hosts_file, self.playbook, '-e',
'{}'.format(extra_vars)]
snapshot_log = open('{}/snapshot.log'.format(result_dir), 'a+')
self.logger.info('Running ansible to create snapshots for: {}'.format(test_name))
subprocess.Popen(subprocess_cmd, stdout=snapshot_log, stderr=subprocess.STDOUT)

35
lib/PerfKit.py
View File

@ -1,5 +1,6 @@
from Connmon import Connmon
from Tools import Tools
from Grafana import Grafana
import glob
import logging
import datetime
@ -8,7 +9,6 @@ import shutil
import subprocess
import time
class PerfKit:
def __init__(self, config):
self.logger = logging.getLogger('browbeat.PerfKit')
@ -16,6 +16,7 @@ class PerfKit:
self.error_count = 0
self.tools = Tools(self.config)
self.connmon = Connmon(self.config)
self.grafana = Grafana(self.config)
self.test_count = 0
self.scenario_count = 0
@ -93,34 +94,10 @@ class PerfKit:
shutil.rmtree("/tmp/perfkitbenchmarker/run_browbeat")
# Grafana integration
if 'grafana' in self.config and self.config['grafana']['enabled']:
grafana_ip = self.config['grafana']['grafana_ip']
grafana_port = self.config['grafana']['grafana_port']
url = 'http://{}:{}/dashboard/db/'.format(grafana_ip, grafana_port)
cloud_name = self.config['grafana']['cloud_name']
for dashboard in self.config['grafana']['dashboards']:
full_url = '{}{}?from={}&to={}&var-Cloud={}'.format(url, dashboard, from_ts, to_ts,
cloud_name)
self.logger.info('{} - Grafana URL: {}'.format(test_name, full_url))
if self.config['grafana']['snapshot']['enabled']:
hosts_file = self.config['ansible']['hosts']
playbook = self.config['ansible']['grafana_snapshot']
extra_vars = 'grafana_ip={} '.format(grafana_ip)
extra_vars += 'grafana_port={} '.format(grafana_port)
extra_vars += 'grafana_api_key={} '.format(self.config['grafana']['snapshot']['grafana_api_key'])
extra_vars += 'from={} '.format(from_ts)
extra_vars += 'to={} '.format(to_ts)
extra_vars += 'results_dir={}/{} '.format(result_dir, test_name)
extra_vars += 'var_cloud={} '.format(cloud_name)
if self.config['grafana']['snapshot']['snapshot_compute']:
extra_vars += 'snapshot_compute=true '
snapshot_cmd = 'ansible-playbook -i {} {} -e "{}"'.format(hosts_file, playbook,
extra_vars)
subprocess_cmd = ['ansible-playbook', '-i', hosts_file, playbook, '-e',
'{}'.format(extra_vars)]
self.logger.info('Snapshot command: {}'.format(snapshot_cmd))
snapshot_log = open('{}/snapshot.log'.format(result_dir), 'a+')
subprocess.Popen(subprocess_cmd, stdout=snapshot_log, stderr=subprocess.STDOUT)
self.grafana.print_dashboard_url(from_ts, to_ts, test_name)
self.grafana.log_snapshot_playbook_cmd(from_ts, to_ts, result_dir, test_name)
self.grafana.run_playbook(from_ts, to_ts, result_dir, test_name)
def start_workloads(self):
self.logger.info("Starting PerfKitBenchmarker Workloads.")

33
lib/Rally.py
View File

@ -1,6 +1,7 @@
from Connmon import Connmon
from Tools import Tools
from collections import OrderedDict
from Grafana import Grafana
import datetime
import glob
import logging
@ -16,6 +17,7 @@ class Rally:
self.config = config
self.tools = Tools(self.config)
self.connmon = Connmon(self.config)
self.grafana = Grafana(self.config)
self.error_count = 0
self.test_count = 0
self.scenario_count = 0
@ -40,34 +42,9 @@ class Rally:
time.sleep(self.config['rally']['sleep_after'])
to_ts = int(time.time() * 1000)
if 'grafana' in self.config and self.config['grafana']['enabled']:
grafana_ip = self.config['grafana']['grafana_ip']
grafana_port = self.config['grafana']['grafana_port']
url = 'http://{}:{}/dashboard/db/'.format(grafana_ip, grafana_port)
cloud_name = self.config['grafana']['cloud_name']
for dashboard in self.config['grafana']['dashboards']:
full_url = '{}{}?from={}&to={}&var-Cloud={}'.format(url, dashboard, from_ts, to_ts,
cloud_name)
self.logger.info('{} - Grafana URL: {}'.format(test_name, full_url))
if self.config['grafana']['snapshot']['enabled']:
hosts_file = self.config['ansible']['hosts']
playbook = self.config['ansible']['grafana_snapshot']
extra_vars = 'grafana_ip={} '.format(grafana_ip)
extra_vars += 'grafana_port={} '.format(grafana_port)
extra_vars += 'grafana_api_key={} '.format(self.config['grafana']['snapshot']['grafana_api_key'])
extra_vars += 'from={} '.format(from_ts)
extra_vars += 'to={} '.format(to_ts)
extra_vars += 'results_dir={}/{} '.format(result_dir, test_name)
extra_vars += 'var_cloud={} '.format(cloud_name)
if self.config['grafana']['snapshot']['snapshot_compute']:
extra_vars += 'snapshot_compute=true '
snapshot_cmd = 'ansible-playbook -i {} {} -e "{}"'.format(hosts_file, playbook,
extra_vars)
subprocess_cmd = ['ansible-playbook', '-i', hosts_file, playbook, '-e',
'{}'.format(extra_vars)]
self.logger.info('Snapshot command: {}'.format(snapshot_cmd))
snapshot_log = open('{}/snapshot.log'.format(result_dir), 'a+')
subprocess.Popen(subprocess_cmd, stdout=snapshot_log, stderr=subprocess.STDOUT)
self.grafana.print_dashboard_url(from_ts, to_ts, test_name)
self.grafana.log_snapshot_playbook_cmd(from_ts, to_ts, result_dir, test_name)
self.grafana.run_playbook(from_ts, to_ts, result_dir, test_name)
def workload_logger(self, result_dir):
base = result_dir.split('/')

14
lib/Shaker.py
View File

@ -1,15 +1,18 @@
from Tools import Tools
from Grafana import Grafana
import yaml
import logging
import datetime
import os
import json
import time
class Shaker:
def __init__(self, config):
self.logger=logging.getLogger('browbeat.Shaker')
self.config = config
self.tools = Tools(self.config)
self.grafana = Grafana(self.config)
self.fail_scenarios = 0
self.pass_scenarios = 0
self.scenarios_count = 0
@ -107,10 +110,19 @@ class Shaker:
" --debug > {4}/{5}.log 2>&1").format(server_endpoint,
port_no, flavor, filename, result_dir, test_name, timeout)
cmd = ("{}; {}").format(cmd_1, cmd_2)
from_ts = int(time.time() * 1000)
if 'sleep_before' in self.config['shaker']:
time.sleep(self.config['shaker']['sleep_before'])
self.tools.run_cmd(cmd)
self.scenarios_count += 1
self.result_check(result_dir, test_name, scenario)
if 'sleep_after' in self.config['shaker']:
time.sleep(self.config['shaker']['sleep_after'])
to_ts = int(time.time() * 1000)
#Snapshotting
self.grafana.print_dashboard_url(from_ts, to_ts, test_name)
self.grafana.log_snapshot_playbook_cmd(from_ts, to_ts, result_dir, test_name)
self.grafana.run_playbook(from_ts, to_ts, result_dir, test_name)
def run_shaker(self):
self.logger.info("Starting Shaker workloads")