Upgrading to Shaker version 0.0.14

Currently we are using Shaker version 0.0.10, however version 0.0.14
has more network tests including ping, tcp bidirectional and also
has a slick format for printing reports for tests with varying
concurrencies.

Moreover, we intend to use the json output from this version to index
shaker results in elastic search through a future commit.

+ Adding dns options
+ Patching validator to check for dns_nameserver key
+ Removing dns option, to be pushed when the pypi package includes
  the change
Change-Id: I522d1c78e7cacda3763f326f5e7dfa53e4ada1a8

ansible/install/roles/browbeat/tasks/main.yml

@@ -86,7 +86,7 @@
   shell: . {{ rally_venv }}/bin/activate; . {{ overcloudrc }}; rally deployment create --fromenv --name overcloud
 
 - name: Install shaker
-  pip: name=pyshaker version=0.0.10 virtualenv={{ shaker_venv }}
+  pip: name=pyshaker version=0.0.14 virtualenv={{ shaker_venv }}
 
 ### begin firewall ###
 # we need TCP/5555 open

ansible/install/shaker_build.yml

@@ -10,8 +10,7 @@
       shell: >
              source {{ overcloudrc }}; source {{ shaker_venv }}/bin/activate;
              shaker-image-builder --flavor-name {{ shaker_flavor }} --image-builder-template
-             {{ shaker_centos }}
+             {{ shaker_centos }} --os-region-name {{ shaker_region }} 
-             --os-region-name {{ shaker_region }}
       become: true
       register: image_result
       failed_when: image_result.rc != 0

browbeat-complete.yaml

@@ -228,6 +228,7 @@ shaker:
   sleep_before: 5
   sleep_after: 5
   venv: /home/stack/shaker-venv
+  dns_nameserver: 8.8.8.8
   shaker_region: regionOne
   scenarios:
     - name: l2-4-1
@@ -236,35 +237,35 @@ shaker:
       compute: 1
       progression: linear
       time: 60
-      file: /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/scenarios/networking/dense_l2.yaml
+      file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml
     - name:  l2-8-1
       enabled: true
       density: 8
       compute: 1
       progression: linear
       time: 60
-      file: /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/scenarios/networking/dense_l2.yaml
+      file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml
     - name: l2-4-2
       enabled: true
       density: 4
       compute: 2
       progression: linear
       time: 60
-      file: /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/scenarios/networking/dense_l2.yaml
+      file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml
     - name: l2-4-8
       enabled: true
       density: 8
       compute: 2
       progression: linear
       time: 60
-      file: /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/scenarios/networking/dense_l2.yaml
+      file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml
     - name: l3-north-south-4-1
       enabled: true
       placement: single_room
       density: 4
       compute: 1
       progression: null
-      file: /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/scenarios/networking/dense_l3_north_south.yaml
+      file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_north_south.yaml
     - name: l3-north-south-8-1
       enabled: false
       placement: single_room
@@ -272,7 +273,7 @@ shaker:
       compute: 1
       progression: null
       time: 60
-      file: /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/scenarios/networking/dense_l3_north_south.yaml
+      file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_north_south.yaml
     - name: l3-north-south-4-2
       enabled: true
       placement: single_room
@@ -280,7 +281,7 @@ shaker:
       compute: 2
       progression: null
       time: 60
-      file: /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/scenarios/networking/dense_l3_north_south.yaml
+      file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_north_south.yaml
     - name: l3-north-south-8-2
       enabled: true
       placement: single_room
@@ -288,32 +289,33 @@ shaker:
       compute: 2
       progression: null
       time: 60
-      file: /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/scenarios/networking/dense_l3_north_south.yaml
+      file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_north_south.yaml
     - name: l3-east-west-4-1
       enabled: true
       density: 4
       compute: 1
       placement: single_room
       time: 60
-      file: /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/scenarios/networking/dense_l3_east_west.yaml
+      file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml
     - name: l3-east-west-8-1
       enabled: true
       density: 8
       compute: 1
       placement: single_room
       time: 60
-      file: /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/scenarios/networking/dense_l3_east_west.yaml
+      file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml
     - name: l3-east-west-4-2
       enabled: true
       density: 4
       compute: 2
       placement: single_room
       time: 60
-      file: /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/scenarios/networking/dense_l3_east_west.yaml
+      file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml
     - name: l3-east-west-8-2
       enabled: true
       density: 8
       compute: 2
       time: 60
       placement: single_room
-      file: /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/scenarios/networking/dense_l3_east_west.yaml
+      file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml
+ 

browbeat-config.yaml

@@ -68,21 +68,21 @@ shaker:
       compute: 1
       progression: linear
       time: 60
-      file: /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/scenarios/networking/dense_l2.yaml
+      file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml
     - name: l3-north-south
-      enabled: false
+      enabled: true
       placement: double_room
       density: 1
       compute: 1
       progression: null
       time: 60
-      file: /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/scenarios/networking/dense_l3_north_south.yaml
+      file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_north_south.yaml
     - name: l3-east-west
-      enabled: false
+      enabled: true
       density: 1
       compute: 1
       time: 60
-      file: /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/scenarios/networking/dense_l3_east_west.yaml
+      file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml
 rally:
   enabled: true
   sleep_before: 5

lib/Shaker.py

@@ -43,17 +43,21 @@ class Shaker(WorkloadBase):
 
     def get_stats(self):
         self.logger.info(
-            "Current number of Shaker tests executed: {}".format(self.test_count))
+            "Current number of Shaker tests executed: {}".format(
+                self.test_count))
         self.logger.info(
-            "Current number of Shaker tests passed: {}".format(self.pass_count))
+            "Current number of Shaker tests passed: {}".format(
+                self.pass_count))
         self.logger.info(
-            "Current number of Shaker tests failed: {}".format(self.error_count))
+            "Current number of Shaker tests failed: {}".format(
+                self.error_count))
 
     def final_stats(self, total):
         self.logger.info(
             "Total Shaker scenarios enabled by user: {}".format(total))
         self.logger.info(
-            "Total number of Shaker tests executed: {}".format(self.test_count))
+            "Total number of Shaker tests executed: {}".format(
+                self.test_count))
         self.logger.info(
             "Total number of Shaker tests passed: {}".format(self.pass_count))
         self.logger.info(
@@ -71,8 +75,7 @@ class Shaker(WorkloadBase):
     def update_scenarios(self):
         self.scenario_count += 1
 
-    def set_scenario(self, scenario):
+    def set_scenario(self, scenario, fname):
-        fname = scenario['file']
         stream = open(fname, 'r')
         data = yaml.load(stream)
         stream.close()
@@ -97,15 +100,18 @@ class Shaker(WorkloadBase):
             data['deployment']['accommodation'][3][
                 'compute_nodes'] = default_compute
         if "progression" in scenario:
-            data['execution']['progression'] = scenario['progression']
+            if scenario['progression'] is None:
+                data['execution'].pop('progression', None)
+            else:
+                data['execution']['progression'] = scenario['progression']
         else:
             data['execution']['progression'] = default_progression
-        data['execution']['tests'] = [d for d in data['execution']
-                                      ['tests'] if d.get('class') == "iperf_graph"]
         if "time" in scenario:
-            data['execution']['tests'][0]['time'] = scenario['time']
+            for test in data['execution']['tests']:
+                test['time'] = scenario['time']
         else:
-            data['execution']['tests'][0]['time'] = default_time
+            for test in data['execution']['tests']:
+                test['time'] = default_time
         with open(fname, 'w') as yaml_file:
             yaml_file.write(yaml.dump(data, default_flow_style=False))
 
@@ -115,55 +121,107 @@ class Shaker(WorkloadBase):
             uuidlist.append(key)
         return uuidlist
 
-    def result_check(self, result_dir, test_name, scenario, to_time, from_time):
+    def result_check(
+            self,
+            result_dir,
+            test_name,
+            scenario,
+            to_time,
+            from_time):
         outputfile = os.path.join(result_dir, test_name + "." + "json")
         error = False
-        with open(outputfile) as data_file:
-            data = json.load(data_file)
-        uuidlist = self.get_uuidlist(data)
         workload = self.__class__.__name__
         new_test_name = test_name.split('-')
         new_test_name = new_test_name[3:]
         new_test_name = '-'.join(new_test_name)
+        try:
+            with open(outputfile) as data_file:
+                data = json.load(data_file)
+        except IOError:
+            self.logger.error(
+                "Cannot open outputfile, possible stack creation failure for test: {}". format(
+                    scenario['name']))
+            self.error_update(
+                result_dir,
+                test_name,
+                scenario,
+                to_time,
+                from_time,
+                new_test_name,
+                workload)
+            return
+        uuidlist = self.get_uuidlist(data)
         for uuid in uuidlist:
             if data['records'][uuid]['status'] != "ok":
                 error = True
         if error:
-            self.logger.error("Failed Test: {}".format(scenario['name']))
+            self.error_update(
-            self.logger.error(
+                result_dir,
-                "saved log to: {}.log".format(os.path.join(result_dir, test_name)))
+                test_name,
-            self.update_fail_tests()
+                scenario,
-            self.update_total_fail_tests()
-            self.get_time_dict(
                 to_time,
                 from_time,
-                scenario['name'],
                 new_test_name,
-                workload,
+                workload)
-                "fail")
         else:
-            self.logger.info("Completed Test: {}".format(scenario['name']))
+            self.success_update(
-            self.logger.info(
+                result_dir,
-                "Saved report to: {}".format(
+                test_name,
-                    os.path.join(
+                scenario,
-                        result_dir,
-                        test_name +
-                        "." +
-                        "html")))
-            self.logger.info(
-                "saved log to: {}.log".format(os.path.join(result_dir, test_name)))
-            self.update_pass_tests()
-            self.update_total_pass_tests()
-            self.get_time_dict(
                 to_time,
                 from_time,
-                scenario['name'],
                 new_test_name,
-                workload,
+                workload)
-                "pass")
 
-    def run_scenario(self, scenario, result_dir, test_name):
+    def error_update(self, result_dir, test_name, scenario, to_time, from_time,
-        filename = scenario['file']
+                     new_test_name, workload):
+        self.logger.error("Failed Test: {}".format(scenario['name']))
+        self.logger.error(
+            "saved log to: {}.log".format(
+                os.path.join(
+                    result_dir,
+                    test_name)))
+        self.update_fail_tests()
+        self.update_total_fail_tests()
+        self.get_time_dict(
+            to_time,
+            from_time,
+            scenario['name'],
+            new_test_name,
+            workload,
+            "fail")
+
+    def success_update(
+            self,
+            result_dir,
+            test_name,
+            scenario,
+            to_time,
+            from_time,
+            new_test_name,
+            workload):
+        self.logger.info("Completed Test: {}".format(scenario['name']))
+        self.logger.info(
+            "Saved report to: {}.html".format(
+                os.path.join(
+                    result_dir,
+                    test_name)))
+        self.logger.info(
+            "saved log to: {}.log".format(
+                os.path.join(
+                    result_dir,
+                    test_name)))
+        self.update_pass_tests()
+        self.update_total_pass_tests()
+        self.get_time_dict(
+            to_time,
+            from_time,
+            scenario['name'],
+            new_test_name,
+            workload,
+            "pass")
+
+    def run_scenario(self, scenario, result_dir, test_name, filename):
         server_endpoint = self.config['shaker']['server']
         port_no = self.config['shaker']['port']
         flavor = self.config['shaker']['flavor']
@@ -176,7 +234,7 @@ class Shaker(WorkloadBase):
             "shaker --server-endpoint {0}:{1} --flavor-name {2} --scenario {3}"
             " --os-region-name {7} --agent-join-timeout {6}"
             " --report {4}/{5}.html --output {4}/{5}.json"
-            " --debug > {4}/{5}.log 2>&1").format(
+            " --book {4}/{5} --debug > {4}/{5}.log 2>&1").format(
             server_endpoint,
             port_no,
             flavor,
@@ -199,7 +257,8 @@ class Shaker(WorkloadBase):
             time.sleep(self.config['shaker']['sleep_after'])
         to_ts = int(time.time() * 1000)
         # Snapshotting
-        self.grafana.print_dashboard_url(from_ts, to_ts, test_name)
+        self.grafana.create_grafana_urls({'from_ts': from_ts, 'to_ts': to_ts})
+        self.grafana.print_dashboard_url(test_name)
         self.grafana.log_snapshot_playbook_cmd(
             from_ts, to_ts, result_dir, test_name)
         self.grafana.run_playbook(from_ts, to_ts, result_dir, test_name)
@@ -209,6 +268,7 @@ class Shaker(WorkloadBase):
         time_stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
         self.logger.debug("Time Stamp (Prefix): {}".format(time_stamp))
         scenarios = self.config.get('shaker')['scenarios']
+        venv = self.config['shaker']['venv']
         self.shaker_checks()
         scen_length = len(scenarios)
         if scen_length > 0:
@@ -217,9 +277,10 @@ class Shaker(WorkloadBase):
                     self.update_scenarios()
                     self.update_total_scenarios()
                     self.logger.info("Scenario: {}".format(scenario['name']))
-                    self.set_scenario(scenario)
+                    fname = os.path.join(venv, scenario['file'])
+                    self.set_scenario(scenario, fname)
                     self.logger.debug("Set Scenario File: {}".format(
-                        scenario['file']))
+                        fname))
                     result_dir = self.tools.create_results_dir(
                         self.config['browbeat'][
                             'results'], time_stamp, "shaker",
@@ -228,9 +289,9 @@ class Shaker(WorkloadBase):
                     self.workload_logger(result_dir, workload)
                     time_stamp1 = datetime.datetime.now().strftime(
                         "%Y%m%d-%H%M%S")
-                    test_name = "{}-browbeat-{}-{}".format(time_stamp1,
+                    test_name = "{}-browbeat-{}-{}".format(
-                                                           "shaker", scenario['name'])
+                        time_stamp1, "shaker", scenario['name'])
-                    self.run_scenario(scenario, result_dir, test_name)
+                    self.run_scenario(scenario, result_dir, test_name, fname)
                     self.get_stats()
                 else:
                     self.logger.info(