diff --git a/browbeat.sh b/browbeat.sh index d4f67a581..713278a44 100755 --- a/browbeat.sh +++ b/browbeat.sh @@ -2,6 +2,9 @@ source ~/stackrc DEBUG=true CONNMON=true +# Number of workers to test. This is a loop. +NUM_WORKERS="24 12 6" +RESET_WORKERS="24" CONNMON_PID=0 CONTROLLERS=$(nova list | grep control) PBENCH=true @@ -18,7 +21,14 @@ TIMES["nova"]=128 declare -A CONCURRENCY CONCURRENCY["keystone"]="64 96 128 160 192 224 256" -CONCURRENCY["nova"]="32 64 128" +CONCURRENCY["nova"]="8 16 32 48" + +ROOT=false +LOGIN_USER="heat-admin" +if [[ $(whoami) == "root" ]]; then + LOGIN_USER="root" + ROOT=true +fi log() { @@ -29,15 +39,15 @@ check_controllers() { for IP in $(echo "$CONTROLLERS" | awk '{print $12}' | cut -d "=" -f 2); do # Number of cores? - CORES=$(ssh -o "${SSH_OPTS}" heat-admin@$IP sudo cat /proc/cpuinfo | grep processor | wc -l) + CORES=$(ssh -o "${SSH_OPTS}" ${LOGIN_USER}@$IP sudo cat /proc/cpuinfo | grep processor | wc -l) log Controller : $IP log Number of cores : $CORES log Service : Keystone - log $(ssh -o "${SSH_OPTS}" heat-admin@$IP sudo cat /etc/keystone/keystone.conf | grep -vi "NONE" | grep -v "#" |grep -E ${WORKERS["keystone"]}) + log $(ssh -o "${SSH_OPTS}" ${LOGIN_USER}@$IP sudo cat /etc/keystone/keystone.conf | grep -vi "NONE" | grep -v "#" |grep -E ${WORKERS["keystone"]}) log Service : Nova - log $(ssh -o "${SSH_OPTS}" heat-admin@$IP sudo cat /etc/nova/nova.conf | grep -vi "NONE" | grep -v "#" |grep -E ${WORKERS["nova"]}) + log $(ssh -o "${SSH_OPTS}" ${LOGIN_USER}@$IP sudo cat /etc/nova/nova.conf | grep -vi "NONE" | grep -v "#" |grep -E ${WORKERS["nova"]}) log Service : Neutron - log $(ssh -o "${SSH_OPTS}" heat-admin@$IP sudo cat /etc/neutron/neutron.conf | grep -vi "NONE" | grep -v "#" |grep -E ${WORKERS["neutron"]}) + log $(ssh -o "${SSH_OPTS}" ${LOGIN_USER}@$IP sudo cat /etc/neutron/neutron.conf | grep -vi "NONE" | grep -v "#" |grep -E ${WORKERS["neutron"]}) done } @@ -68,40 +78,40 @@ update_workers() for IP in $(echo "$CONTROLLERS" | awk '{print $12}' | cut -d "=" -f 2); do for i in $(echo ${WORKERS[$osp_service]} | tr "|" "\n") ; do log Copying Config files to : $IP - ssh -o "${SSH_OPTS}" heat-admin@$IP sudo cp ${services[$osp_service]} ${services[$osp_service]}-copy - ssh -o "${SSH_OPTS}" heat-admin@$IP sudo "sed -i -e \"s/^\(${i}\)\( \)*=\( \)*\([0-9]\)*/${i}=${wkr_count}/g\" ${services[$osp_service]}" + ssh -o "${SSH_OPTS}" ${LOGIN_USER}@$IP sudo cp ${services[$osp_service]} ${services[$osp_service]}-copy + ssh -o "${SSH_OPTS}" ${LOGIN_USER}@$IP sudo "sed -i -e \"s/^\(${i}\)\( \)*=\( \)*\([0-9]\)*/${i}=${wkr_count}/g\" ${services[$osp_service]}" done done if [ "${osp_service}" == "keystone" ]; then IP=`echo "$CONTROLLERS" | head -n 1 | awk '{print $12}' | cut -d "=" -f 2` - ssh -o "${SSH_OPTS}" heat-admin@$IP sudo "pcs resource unmanage openstack-keystone" + ssh -o "${SSH_OPTS}" ${LOGIN_USER}@$IP sudo "pcs resource unmanage openstack-keystone" for IP in $(echo "$CONTROLLERS" | awk '{print $12}' | cut -d "=" -f 2); do - ssh -o "${SSH_OPTS}" heat-admin@$IP sudo "systemctl restart openstack-keystone" + ssh -o "${SSH_OPTS}" ${LOGIN_USER}@$IP sudo "systemctl restart openstack-keystone" done - ssh -o "${SSH_OPTS}" heat-admin@$IP sudo "pcs resource manage openstack-keystone" + ssh -o "${SSH_OPTS}" ${LOGIN_USER}@$IP sudo "pcs resource manage openstack-keystone" fi if [ "${osp_service}" == "nova" ]; then IP=`echo "$CONTROLLERS" | head -n 1 | awk '{print $12}' | cut -d "=" -f 2` - ssh -o "${SSH_OPTS}" heat-admin@$IP sudo "pcs resource unmanage openstack-nova-api" - ssh -o "${SSH_OPTS}" heat-admin@$IP sudo "pcs resource unmanage openstack-nova-conductor" - ssh -o "${SSH_OPTS}" heat-admin@$IP sudo "pcs resource unmanage openstack-nova-scheduler" + ssh -o "${SSH_OPTS}" ${LOGIN_USER}@$IP sudo "pcs resource unmanage openstack-nova-api" + ssh -o "${SSH_OPTS}" ${LOGIN_USER}@$IP sudo "pcs resource unmanage openstack-nova-conductor" + ssh -o "${SSH_OPTS}" ${LOGIN_USER}@$IP sudo "pcs resource unmanage openstack-nova-scheduler" for IP in $(echo "$CONTROLLERS" | awk '{print $12}' | cut -d "=" -f 2); do - ssh -o "${SSH_OPTS}" heat-admin@$IP sudo "systemctl restart openstack-nova-api" - ssh -o "${SSH_OPTS}" heat-admin@$IP sudo "systemctl restart openstack-nova-conductor" - ssh -o "${SSH_OPTS}" heat-admin@$IP sudo "systemctl restart openstack-nova-scheduler" + ssh -o "${SSH_OPTS}" ${LOGIN_USER}@$IP sudo "systemctl restart openstack-nova-api" + ssh -o "${SSH_OPTS}" ${LOGIN_USER}@$IP sudo "systemctl restart openstack-nova-conductor" + ssh -o "${SSH_OPTS}" ${LOGIN_USER}@$IP sudo "systemctl restart openstack-nova-scheduler" done - ssh -o "${SSH_OPTS}" heat-admin@$IP sudo "pcs resource manage openstack-nova-api" - ssh -o "${SSH_OPTS}" heat-admin@$IP sudo "pcs resource manage openstack-nova-conductor" - ssh -o "${SSH_OPTS}" heat-admin@$IP sudo "pcs resource manage openstack-nova-scheduler" + ssh -o "${SSH_OPTS}" ${LOGIN_USER}@$IP sudo "pcs resource manage openstack-nova-api" + ssh -o "${SSH_OPTS}" ${LOGIN_USER}@$IP sudo "pcs resource manage openstack-nova-conductor" + ssh -o "${SSH_OPTS}" ${LOGIN_USER}@$IP sudo "pcs resource manage openstack-nova-scheduler" fi sleep 5 # Give things time to come up for IP in $(echo "$CONTROLLERS" | awk '{print $12}' | cut -d "=" -f 2); do log Validate number of workers - keystone_num=$(ssh -o "${SSH_OPTS}" heat-admin@$IP sudo ps afx | grep "[Kk]eystone" | wc -l) - nova_num=$(ssh -o "${SSH_OPTS}" heat-admin@$IP sudo ps afx | grep "[Nn]ova" | wc -l) + keystone_num=$(ssh -o "${SSH_OPTS}" ${LOGIN_USER}@$IP sudo ps afx | grep "[Kk]eystone" | wc -l) + nova_num=$(ssh -o "${SSH_OPTS}" ${LOGIN_USER}@$IP sudo ps afx | grep "[Nn]ova" | wc -l) log $IP : keystone : $keystone_num workers log $IP : nova : $nova_num workers # Keystone should be 2x for public and private + 1 for main process @@ -195,11 +205,12 @@ setup_pbench() sudo /opt/pbench-agent/util-scripts/register-tool --name=vmstat -- --interval=${PBENCH_INTERVAL} sudo /opt/pbench-agent/util-scripts/register-tool --name=pidstat -- --interval=${PBENCH_INTERVAL} for IP in $(echo "$CONTROLLERS" | awk '{print $12}' | cut -d "=" -f 2); do - register-tool --name=mpstat --remote=${IP} -- --interval=${PBENCH_INTERVAL} - register-tool --name=iostat --remote=${IP} -- --interval=${PBENCH_INTERVAL} - register-tool --name=sar --remote=${IP} -- --interval=${PBENCH_INTERVAL} - register-tool --name=vmstat --remote=${IP} -- --interval=${PBENCH_INTERVAL} - register-tool --name=pidstat --remote=${IP} -- --interval=${PBENCH_INTERVAL} + sudo /opt/pbench-agent/util-scripts/register-tool --name=mpstat --remote=${IP} -- --interval=${PBENCH_INTERVAL} + sudo /opt/pbench-agent/util-scripts/register-tool --name=iostat --remote=${IP} -- --interval=${PBENCH_INTERVAL} + sudo /opt/pbench-agent/util-scripts/register-tool --name=sar --remote=${IP} -- --interval=${PBENCH_INTERVAL} + sudo /opt/pbench-agent/util-scripts/register-tool --name=vmstat --remote=${IP} -- --interval=${PBENCH_INTERVAL} + sudo /opt/pbench-agent/util-scripts/register-tool --name=pidstat --remote=${IP} -- --interval=${PBENCH_INTERVAL} + sudo /opt/pbench-agent/util-scripts/register-tool --name=user-tool --remote=${IP} -- --tool-name=mariadb-conntrack --start-script=/opt/usertool/mariadb-track.sh done } @@ -207,7 +218,7 @@ truncate_token_bloat() { log "Truncating Token Bloat" IP=`echo "$CONTROLLERS" | head -n 1 | awk '{print $12}' | cut -d "=" -f 2` - ssh -o "${SSH_OPTS}" heat-admin@$IP sudo "mysql keystone -e 'truncate token;'" + ssh -o "${SSH_OPTS}" ${LOGIN_USER}@$IP sudo "mysql keystone -e 'truncate token;'" } @@ -230,7 +241,8 @@ fi mkdir -p results check_controllers -for num_wkrs in `seq 24 -4 4`; do +#for num_wkrs in `seq 24 -4 4`; do +for num_wkrs in ${NUM_WORKERS} ; do num_wkr_padded="$(printf "%02d" ${num_wkrs})" update_workers ${num_wkrs} keystone @@ -242,6 +254,6 @@ for num_wkrs in `seq 24 -4 4`; do run_rally nova "${complete_test_prefix}-nova-${num_wkr_padded}" done -update_workers 24 keystone -update_workers 24 nova +update_workers ${RESET_WORKERS} keystone +update_workers ${RESET_WORKERS} nova check_controllers diff --git a/graphing/pyplot.py b/graphing/pyplot.py index 750412a31..d1100d74a 100644 --- a/graphing/pyplot.py +++ b/graphing/pyplot.py @@ -26,38 +26,48 @@ for row in reader: data[service][row['hostname']]['max_connections'] = [] data[service][row['hostname']]['connection_count'] = [] data[service][row['hostname']]['checkout_count'] = [] + data[service][row['hostname']]['max_checkedout'] = [] if row['progname'] == service : data[service][row['hostname']]['timestamp'].append(datetime.strptime(row['timestamp'],'%Y-%m-%d %H:%M:%S')) data[service][row['hostname']]['connection_count'].append(row['connection_count']) data[service][row['hostname']]['max_connections'].append(row['max_connections']) data[service][row['hostname']]['checkout_count'].append(row['checkout_count']) + data[service][row['hostname']]['max_checkedout'].append(row['max_checkedout']) for service in data : pos=0 - for host in data[service] : - plt.title("Service : %s" % service) - plt.xlabel("Time") - plt.ylabel("Max Connections") - controller,=plt.plot_date(data[service][host]['timestamp'][1::30],data[service][host]['connection_count'][1::30],'c',linewidth=2,label="%s-controller0"%service) - controller.set_color(color_wheel[pos]) - pos=pos+1 +# for host in data[service] : +# plt.title("Service : %s" % service) +# plt.xlabel("Time") +# plt.ylabel("Max Connections") +# controller,=plt.plot_date(data[service][host]['timestamp'][1::30],data[service][host]['connection_count'][1::30],'c',linewidth=2,label="%s-controller0"%service) +# controller,=plt.plot_date(data[service][host]['timestamp'],data[service][host]['connection_count'],'c',linewidth=2,label="%s-controller0"%service) +# controller.set_color(color_wheel[pos]) +# pos=pos+1 - plt.legend(["%s-controller0"%service,"%s-controller1"%service,"%s-controller2"%service]) - plt.savefig("%s_%s-connction_count.png"%(sys.argv[1],ntpath.basename(service)), bbox_inches='tight') - plt.close() +# plt.legend(["%s-controller0"%service,"%s-controller1"%service,"%s-controller2"%service]) +# plt.savefig("%s_%s-connction_count.png"%(sys.argv[1],ntpath.basename(service)), bbox_inches='tight') +# plt.close() pos=0 for host in data[service] : plt.title("Service : %s" % service) plt.xlabel("Time") plt.ylabel("Connections") - controller,=plt.plot_date(data[service][host]['timestamp'][1::30],data[service][host]['connection_count'][1::30],'c',linewidth=4,label="%s-controller0-conn"%service) - controller1,=plt.plot_date(data[service][host]['timestamp'][1::30],data[service][host]['checkout_count'][1::30],'c',linewidth=1,label="%s-controller0-ckout"%service) +# controller,=plt.plot_date(data[service][host]['timestamp'][1::30],data[service][host]['connection_count'][1::30],'c',linewidth=4,label="%s-controller0-conn"%service) +# controller1,=plt.plot_date(data[service][host]['timestamp'][1::30],data[service][host]['checkout_count'][1::30],'c',linewidth=2,label="%s-controller0-ckout"%service) +# controller2,=plt.plot_date(data[service][host]['timestamp'][1::30],data[service][host]['max_checkedout'][1::30],'c',linewidth=1,linestyle='dashdot',label="%s-controller0-ckout"%service,) + + controller,=plt.plot_date(data[service][host]['timestamp'],data[service][host]['connection_count'],'c',linewidth=4,label="%s-controller0-conn"%service) + controller1,=plt.plot_date(data[service][host]['timestamp'],data[service][host]['checkout_count'],'c',linewidth=2,label="%s-controller0-ckout"%service) + controller2,=plt.plot_date(data[service][host]['timestamp'],data[service][host]['max_checkedout'],'c',linewidth=1,linestyle='dashdot',label="%s-controller0-ckout"%service,) controller.set_color(color_wheel[pos]) controller1.set_color(color_wheel[pos]) + controller2.set_color(color_wheel[pos]) pos=pos+1 - plt.legend(["%s-controller0-conn"%service,"%s-controller0-ckout"%service,"%s-controller1-conn"%service,"%s-controller1-ckout"%service,"%s-controller2-conn"%service,"%s-controller2-ckout"%service]) + plt.legend(["%s-controller0-conn"%service,"%s-controller0-ckout"%service,"%s-controller0-max-ckout"%service,"%s-controller1-conn"%service,"%s-controller1-ckout"%service,"%s-controller1-max-ckout"%service,"%s-controller2-conn"%service,"%s-controller2-ckout"%service,"%s-controller2-max-ckout"%service]) +# plt.legend(["%s-controller0-ckout"%service,"%s-controller0-max-ckout"%service,"%s-controller1-ckout"%service,"%s-controller1-max-ckout"%service,"%s-controller2-ckout"%service,"%s-controller2-max-ckout"%service]) plt.savefig("%s_%s-connctions.png"%(sys.argv[1],ntpath.basename(service)), bbox_inches='tight') plt.close() diff --git a/pbench/mariadb-track.sh b/pbench/mariadb-track.sh new file mode 100755 index 000000000..bb117bad7 --- /dev/null +++ b/pbench/mariadb-track.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +CONNECTIONS=$(mysql -u root -e "select count(*), db from information_schema.processlist group by db") +DATA_FILE=./pbench-maraidb-connections.csv +INTERVAL=10 +while true; do + TIME=$(date "+%y-%m-%d %H:%M:%S") + echo "${CONNECTIONS}" | while IFS= read -r line ; do + if [[ "$line" =~ .*count.* ]] ; then continue ; fi + if [[ "$line" =~ .*NULL.* ]] ; then continue ; fi + count=$(echo $line | awk '{print $1}') + service=$(echo $line | awk '{print $2}') + echo "$TIME,$service,$count" + echo "$TIME,$service,$count" >> ${DATA_FILE} + done + sleep $INTERVAL +done diff --git a/postprocess/csv-to-json.py b/postprocess/csv-to-json.py new file mode 100644 index 000000000..048935c6d --- /dev/null +++ b/postprocess/csv-to-json.py @@ -0,0 +1,33 @@ +import csv +import json +import sys +import getopt +from collections import defaultdict + +def main(argv): + inputfile = None + + try: + opts, args = getopt.getopt(argv,"c",["csv="]) + except : + print "csv-to-json.py --csv=" + sys.exit(2) + + for opt, arg in opts: + if opt == '-h': + print 'csv-to-json.py --csv=' + sys.exit() + elif opt in ("-c", "--csv"): + inputfile = arg + if inputfile == None : + print "Error : No input file passed" + print "Usage:" + print "csv-to-json.py --csv=" + sys.exit(2) + + print "Opening %s" % inputfile + csvFile = open(inputfile) + data = list(csv.reader(csvFile)) + +if __name__ == "__main__": + main(sys.argv[1:])