Upgrading to Shaker version 0.0.14

Currently we are using Shaker version 0.0.10, however version 0.0.14
has more network tests including ping, tcp bidirectional and also
has a slick format for printing reports for tests with varying
concurrencies.

Moreover, we intend to use the json output from this version to index
shaker results in elastic search through a future commit.

+ Adding dns options
+ Patching validator to check for dns_nameserver key
+ Removing dns option, to be pushed when the pypi package includes
  the change
Change-Id: I522d1c78e7cacda3763f326f5e7dfa53e4ada1a8
This commit is contained in:
Sindhur 2016-06-14 17:20:27 -04:00 committed by Sai Sindhur Malleni
parent 77aa0db423
commit e76b8acb4b
5 changed files with 131 additions and 69 deletions

View File

@ -86,7 +86,7 @@
shell: . {{ rally_venv }}/bin/activate; . {{ overcloudrc }}; rally deployment create --fromenv --name overcloud shell: . {{ rally_venv }}/bin/activate; . {{ overcloudrc }}; rally deployment create --fromenv --name overcloud
- name: Install shaker - name: Install shaker
pip: name=pyshaker version=0.0.10 virtualenv={{ shaker_venv }} pip: name=pyshaker version=0.0.14 virtualenv={{ shaker_venv }}
### begin firewall ### ### begin firewall ###
# we need TCP/5555 open # we need TCP/5555 open

View File

@ -10,8 +10,7 @@
shell: > shell: >
source {{ overcloudrc }}; source {{ shaker_venv }}/bin/activate; source {{ overcloudrc }}; source {{ shaker_venv }}/bin/activate;
shaker-image-builder --flavor-name {{ shaker_flavor }} --image-builder-template shaker-image-builder --flavor-name {{ shaker_flavor }} --image-builder-template
{{ shaker_centos }} {{ shaker_centos }} --os-region-name {{ shaker_region }}
--os-region-name {{ shaker_region }}
become: true become: true
register: image_result register: image_result
failed_when: image_result.rc != 0 failed_when: image_result.rc != 0

View File

@ -228,6 +228,7 @@ shaker:
sleep_before: 5 sleep_before: 5
sleep_after: 5 sleep_after: 5
venv: /home/stack/shaker-venv venv: /home/stack/shaker-venv
dns_nameserver: 8.8.8.8
shaker_region: regionOne shaker_region: regionOne
scenarios: scenarios:
- name: l2-4-1 - name: l2-4-1
@ -236,35 +237,35 @@ shaker:
compute: 1 compute: 1
progression: linear progression: linear
time: 60 time: 60
file: /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/scenarios/networking/dense_l2.yaml file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml
- name: l2-8-1 - name: l2-8-1
enabled: true enabled: true
density: 8 density: 8
compute: 1 compute: 1
progression: linear progression: linear
time: 60 time: 60
file: /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/scenarios/networking/dense_l2.yaml file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml
- name: l2-4-2 - name: l2-4-2
enabled: true enabled: true
density: 4 density: 4
compute: 2 compute: 2
progression: linear progression: linear
time: 60 time: 60
file: /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/scenarios/networking/dense_l2.yaml file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml
- name: l2-4-8 - name: l2-4-8
enabled: true enabled: true
density: 8 density: 8
compute: 2 compute: 2
progression: linear progression: linear
time: 60 time: 60
file: /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/scenarios/networking/dense_l2.yaml file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml
- name: l3-north-south-4-1 - name: l3-north-south-4-1
enabled: true enabled: true
placement: single_room placement: single_room
density: 4 density: 4
compute: 1 compute: 1
progression: null progression: null
file: /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/scenarios/networking/dense_l3_north_south.yaml file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_north_south.yaml
- name: l3-north-south-8-1 - name: l3-north-south-8-1
enabled: false enabled: false
placement: single_room placement: single_room
@ -272,7 +273,7 @@ shaker:
compute: 1 compute: 1
progression: null progression: null
time: 60 time: 60
file: /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/scenarios/networking/dense_l3_north_south.yaml file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_north_south.yaml
- name: l3-north-south-4-2 - name: l3-north-south-4-2
enabled: true enabled: true
placement: single_room placement: single_room
@ -280,7 +281,7 @@ shaker:
compute: 2 compute: 2
progression: null progression: null
time: 60 time: 60
file: /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/scenarios/networking/dense_l3_north_south.yaml file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_north_south.yaml
- name: l3-north-south-8-2 - name: l3-north-south-8-2
enabled: true enabled: true
placement: single_room placement: single_room
@ -288,32 +289,33 @@ shaker:
compute: 2 compute: 2
progression: null progression: null
time: 60 time: 60
file: /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/scenarios/networking/dense_l3_north_south.yaml file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_north_south.yaml
- name: l3-east-west-4-1 - name: l3-east-west-4-1
enabled: true enabled: true
density: 4 density: 4
compute: 1 compute: 1
placement: single_room placement: single_room
time: 60 time: 60
file: /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/scenarios/networking/dense_l3_east_west.yaml file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml
- name: l3-east-west-8-1 - name: l3-east-west-8-1
enabled: true enabled: true
density: 8 density: 8
compute: 1 compute: 1
placement: single_room placement: single_room
time: 60 time: 60
file: /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/scenarios/networking/dense_l3_east_west.yaml file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml
- name: l3-east-west-4-2 - name: l3-east-west-4-2
enabled: true enabled: true
density: 4 density: 4
compute: 2 compute: 2
placement: single_room placement: single_room
time: 60 time: 60
file: /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/scenarios/networking/dense_l3_east_west.yaml file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml
- name: l3-east-west-8-2 - name: l3-east-west-8-2
enabled: true enabled: true
density: 8 density: 8
compute: 2 compute: 2
time: 60 time: 60
placement: single_room placement: single_room
file: /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/scenarios/networking/dense_l3_east_west.yaml file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml

View File

@ -68,21 +68,21 @@ shaker:
compute: 1 compute: 1
progression: linear progression: linear
time: 60 time: 60
file: /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/scenarios/networking/dense_l2.yaml file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml
- name: l3-north-south - name: l3-north-south
enabled: false enabled: true
placement: double_room placement: double_room
density: 1 density: 1
compute: 1 compute: 1
progression: null progression: null
time: 60 time: 60
file: /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/scenarios/networking/dense_l3_north_south.yaml file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_north_south.yaml
- name: l3-east-west - name: l3-east-west
enabled: false enabled: true
density: 1 density: 1
compute: 1 compute: 1
time: 60 time: 60
file: /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/scenarios/networking/dense_l3_east_west.yaml file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml
rally: rally:
enabled: true enabled: true
sleep_before: 5 sleep_before: 5

View File

@ -43,17 +43,21 @@ class Shaker(WorkloadBase):
def get_stats(self): def get_stats(self):
self.logger.info( self.logger.info(
"Current number of Shaker tests executed: {}".format(self.test_count)) "Current number of Shaker tests executed: {}".format(
self.test_count))
self.logger.info( self.logger.info(
"Current number of Shaker tests passed: {}".format(self.pass_count)) "Current number of Shaker tests passed: {}".format(
self.pass_count))
self.logger.info( self.logger.info(
"Current number of Shaker tests failed: {}".format(self.error_count)) "Current number of Shaker tests failed: {}".format(
self.error_count))
def final_stats(self, total): def final_stats(self, total):
self.logger.info( self.logger.info(
"Total Shaker scenarios enabled by user: {}".format(total)) "Total Shaker scenarios enabled by user: {}".format(total))
self.logger.info( self.logger.info(
"Total number of Shaker tests executed: {}".format(self.test_count)) "Total number of Shaker tests executed: {}".format(
self.test_count))
self.logger.info( self.logger.info(
"Total number of Shaker tests passed: {}".format(self.pass_count)) "Total number of Shaker tests passed: {}".format(self.pass_count))
self.logger.info( self.logger.info(
@ -71,8 +75,7 @@ class Shaker(WorkloadBase):
def update_scenarios(self): def update_scenarios(self):
self.scenario_count += 1 self.scenario_count += 1
def set_scenario(self, scenario): def set_scenario(self, scenario, fname):
fname = scenario['file']
stream = open(fname, 'r') stream = open(fname, 'r')
data = yaml.load(stream) data = yaml.load(stream)
stream.close() stream.close()
@ -97,15 +100,18 @@ class Shaker(WorkloadBase):
data['deployment']['accommodation'][3][ data['deployment']['accommodation'][3][
'compute_nodes'] = default_compute 'compute_nodes'] = default_compute
if "progression" in scenario: if "progression" in scenario:
data['execution']['progression'] = scenario['progression'] if scenario['progression'] is None:
data['execution'].pop('progression', None)
else:
data['execution']['progression'] = scenario['progression']
else: else:
data['execution']['progression'] = default_progression data['execution']['progression'] = default_progression
data['execution']['tests'] = [d for d in data['execution']
['tests'] if d.get('class') == "iperf_graph"]
if "time" in scenario: if "time" in scenario:
data['execution']['tests'][0]['time'] = scenario['time'] for test in data['execution']['tests']:
test['time'] = scenario['time']
else: else:
data['execution']['tests'][0]['time'] = default_time for test in data['execution']['tests']:
test['time'] = default_time
with open(fname, 'w') as yaml_file: with open(fname, 'w') as yaml_file:
yaml_file.write(yaml.dump(data, default_flow_style=False)) yaml_file.write(yaml.dump(data, default_flow_style=False))
@ -115,55 +121,107 @@ class Shaker(WorkloadBase):
uuidlist.append(key) uuidlist.append(key)
return uuidlist return uuidlist
def result_check(self, result_dir, test_name, scenario, to_time, from_time): def result_check(
self,
result_dir,
test_name,
scenario,
to_time,
from_time):
outputfile = os.path.join(result_dir, test_name + "." + "json") outputfile = os.path.join(result_dir, test_name + "." + "json")
error = False error = False
with open(outputfile) as data_file:
data = json.load(data_file)
uuidlist = self.get_uuidlist(data)
workload = self.__class__.__name__ workload = self.__class__.__name__
new_test_name = test_name.split('-') new_test_name = test_name.split('-')
new_test_name = new_test_name[3:] new_test_name = new_test_name[3:]
new_test_name = '-'.join(new_test_name) new_test_name = '-'.join(new_test_name)
try:
with open(outputfile) as data_file:
data = json.load(data_file)
except IOError:
self.logger.error(
"Cannot open outputfile, possible stack creation failure for test: {}". format(
scenario['name']))
self.error_update(
result_dir,
test_name,
scenario,
to_time,
from_time,
new_test_name,
workload)
return
uuidlist = self.get_uuidlist(data)
for uuid in uuidlist: for uuid in uuidlist:
if data['records'][uuid]['status'] != "ok": if data['records'][uuid]['status'] != "ok":
error = True error = True
if error: if error:
self.logger.error("Failed Test: {}".format(scenario['name'])) self.error_update(
self.logger.error( result_dir,
"saved log to: {}.log".format(os.path.join(result_dir, test_name))) test_name,
self.update_fail_tests() scenario,
self.update_total_fail_tests()
self.get_time_dict(
to_time, to_time,
from_time, from_time,
scenario['name'],
new_test_name, new_test_name,
workload, workload)
"fail")
else: else:
self.logger.info("Completed Test: {}".format(scenario['name'])) self.success_update(
self.logger.info( result_dir,
"Saved report to: {}".format( test_name,
os.path.join( scenario,
result_dir,
test_name +
"." +
"html")))
self.logger.info(
"saved log to: {}.log".format(os.path.join(result_dir, test_name)))
self.update_pass_tests()
self.update_total_pass_tests()
self.get_time_dict(
to_time, to_time,
from_time, from_time,
scenario['name'],
new_test_name, new_test_name,
workload, workload)
"pass")
def run_scenario(self, scenario, result_dir, test_name): def error_update(self, result_dir, test_name, scenario, to_time, from_time,
filename = scenario['file'] new_test_name, workload):
self.logger.error("Failed Test: {}".format(scenario['name']))
self.logger.error(
"saved log to: {}.log".format(
os.path.join(
result_dir,
test_name)))
self.update_fail_tests()
self.update_total_fail_tests()
self.get_time_dict(
to_time,
from_time,
scenario['name'],
new_test_name,
workload,
"fail")
def success_update(
self,
result_dir,
test_name,
scenario,
to_time,
from_time,
new_test_name,
workload):
self.logger.info("Completed Test: {}".format(scenario['name']))
self.logger.info(
"Saved report to: {}.html".format(
os.path.join(
result_dir,
test_name)))
self.logger.info(
"saved log to: {}.log".format(
os.path.join(
result_dir,
test_name)))
self.update_pass_tests()
self.update_total_pass_tests()
self.get_time_dict(
to_time,
from_time,
scenario['name'],
new_test_name,
workload,
"pass")
def run_scenario(self, scenario, result_dir, test_name, filename):
server_endpoint = self.config['shaker']['server'] server_endpoint = self.config['shaker']['server']
port_no = self.config['shaker']['port'] port_no = self.config['shaker']['port']
flavor = self.config['shaker']['flavor'] flavor = self.config['shaker']['flavor']
@ -176,7 +234,7 @@ class Shaker(WorkloadBase):
"shaker --server-endpoint {0}:{1} --flavor-name {2} --scenario {3}" "shaker --server-endpoint {0}:{1} --flavor-name {2} --scenario {3}"
" --os-region-name {7} --agent-join-timeout {6}" " --os-region-name {7} --agent-join-timeout {6}"
" --report {4}/{5}.html --output {4}/{5}.json" " --report {4}/{5}.html --output {4}/{5}.json"
" --debug > {4}/{5}.log 2>&1").format( " --book {4}/{5} --debug > {4}/{5}.log 2>&1").format(
server_endpoint, server_endpoint,
port_no, port_no,
flavor, flavor,
@ -199,7 +257,8 @@ class Shaker(WorkloadBase):
time.sleep(self.config['shaker']['sleep_after']) time.sleep(self.config['shaker']['sleep_after'])
to_ts = int(time.time() * 1000) to_ts = int(time.time() * 1000)
# Snapshotting # Snapshotting
self.grafana.print_dashboard_url(from_ts, to_ts, test_name) self.grafana.create_grafana_urls({'from_ts': from_ts, 'to_ts': to_ts})
self.grafana.print_dashboard_url(test_name)
self.grafana.log_snapshot_playbook_cmd( self.grafana.log_snapshot_playbook_cmd(
from_ts, to_ts, result_dir, test_name) from_ts, to_ts, result_dir, test_name)
self.grafana.run_playbook(from_ts, to_ts, result_dir, test_name) self.grafana.run_playbook(from_ts, to_ts, result_dir, test_name)
@ -209,6 +268,7 @@ class Shaker(WorkloadBase):
time_stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") time_stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
self.logger.debug("Time Stamp (Prefix): {}".format(time_stamp)) self.logger.debug("Time Stamp (Prefix): {}".format(time_stamp))
scenarios = self.config.get('shaker')['scenarios'] scenarios = self.config.get('shaker')['scenarios']
venv = self.config['shaker']['venv']
self.shaker_checks() self.shaker_checks()
scen_length = len(scenarios) scen_length = len(scenarios)
if scen_length > 0: if scen_length > 0:
@ -217,9 +277,10 @@ class Shaker(WorkloadBase):
self.update_scenarios() self.update_scenarios()
self.update_total_scenarios() self.update_total_scenarios()
self.logger.info("Scenario: {}".format(scenario['name'])) self.logger.info("Scenario: {}".format(scenario['name']))
self.set_scenario(scenario) fname = os.path.join(venv, scenario['file'])
self.set_scenario(scenario, fname)
self.logger.debug("Set Scenario File: {}".format( self.logger.debug("Set Scenario File: {}".format(
scenario['file'])) fname))
result_dir = self.tools.create_results_dir( result_dir = self.tools.create_results_dir(
self.config['browbeat'][ self.config['browbeat'][
'results'], time_stamp, "shaker", 'results'], time_stamp, "shaker",
@ -228,9 +289,9 @@ class Shaker(WorkloadBase):
self.workload_logger(result_dir, workload) self.workload_logger(result_dir, workload)
time_stamp1 = datetime.datetime.now().strftime( time_stamp1 = datetime.datetime.now().strftime(
"%Y%m%d-%H%M%S") "%Y%m%d-%H%M%S")
test_name = "{}-browbeat-{}-{}".format(time_stamp1, test_name = "{}-browbeat-{}-{}".format(
"shaker", scenario['name']) time_stamp1, "shaker", scenario['name'])
self.run_scenario(scenario, result_dir, test_name) self.run_scenario(scenario, result_dir, test_name, fname)
self.get_stats() self.get_stats()
else: else:
self.logger.info( self.logger.info(