Auto-Detect Shaker File paths

We can no longer hardcode shaker file paths due to python2.7/python3.6 variability.

Change-Id: I2ca509c896c66b230cf975b74c3988a6bec73077

browbeat-complete.yaml

@@ -277,6 +277,7 @@ workloads:
         browbeat_ssh_config: ansible/ssh-config
         username: heat-admin
 
+# file for shaker should be one of https://opendev.org/performa/shaker/src/branch/master/shaker/scenarios/openstack
 # Shaker scenarios require atleast 2 compute nodes
   - name: l2-4-1
     enabled: true
@@ -285,7 +286,7 @@ workloads:
     compute: 1
     progression: linear
     time: 60
-    file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml
+    file: dense_l2.yaml
   - name:  l2-8-1
     enabled: true
     type: shaker
@@ -293,7 +294,7 @@ workloads:
     compute: 1
     progression: linear
     time: 60
-    file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml
+    file: dense_l2.yaml
   - name: l2-4-2
     enabled: true
     type: shaker
@@ -301,7 +302,7 @@ workloads:
     compute: 2
     progression: linear
     time: 60
-    file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml
+    file: dense_l2.yaml
   - name: l2-8-2
     enabled: true
     type: shaker
@@ -309,7 +310,7 @@ workloads:
     compute: 2
     progression: linear
     time: 60
-    file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml
+    file: dense_l2.yaml
   - name: l3-north-south-4-1
     enabled: true
     type: shaker
@@ -317,7 +318,7 @@ workloads:
     density: 4
     compute: 1
     progression: null
-    file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_north_south.yaml
+    file: dense_l3_north_south.yaml
   - name: l3-north-south-8-1
     enabled: false
     type: shaker
@@ -326,7 +327,7 @@ workloads:
     compute: 1
     progression: null
     time: 60
-    file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_north_south.yaml
+    file: dense_l3_north_south.yaml
   - name: l3-north-south-4-2
     enabled: true
     type: shaker
@@ -335,7 +336,7 @@ workloads:
     compute: 2
     progression: null
     time: 60
-    file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_north_south.yaml
+    file: dense_l3_north_south.yaml
   - name: l3-north-south-8-2
     enabled: true
     type: shaker
@@ -344,7 +345,7 @@ workloads:
     compute: 2
     progression: null
     time: 60
-    file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_north_south.yaml
+    file: dense_l3_north_south.yaml
   - name: l3-east-west-4-1
     enabled: true
     type: shaker
@@ -352,7 +353,7 @@ workloads:
     compute: 1
     placement: single_room
     time: 60
-    file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml
+    file: dense_l3_east_west.yaml
   - name: l3-east-west-8-1
     enabled: true
     type: shaker
@@ -360,7 +361,7 @@ workloads:
     compute: 1
     placement: single_room
     time: 60
-    file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml
+    file: dense_l3_east_west.yaml
   - name: l3-east-west-4-2
     enabled: true
     type: shaker
@@ -368,7 +369,7 @@ workloads:
     compute: 2
     placement: single_room
     time: 60
-    file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml
+    file: dense_l3_east_west.yaml
   - name: l3-east-west-8-2
     enabled: true
     type: shaker
@@ -376,4 +377,4 @@ workloads:
     compute: 2
     time: 60
     placement: single_room
-    file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml
+    file: dense_l3_east_west.yaml

browbeat-config.yaml

@@ -68,6 +68,7 @@ workloads:
     type: perfkit
     benchmarks: fio
     openstack_volume_size: 1
+# file for shaker should be one of https://opendev.org/performa/shaker/src/branch/master/shaker/scenarios/openstack
 # Shaker
   - name: shaker-l2
     enabled: true
@@ -76,7 +77,7 @@ workloads:
     compute: 1
     progression: linear
     time: 60
-    file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml
+    file: dense_l2.yaml
   - name: shaker-l3-north-south
     enabled: true
     type: shaker
@@ -85,14 +86,14 @@ workloads:
     compute: 1
     progression: null
     time: 60
-    file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_north_south.yaml
+    file: dense_l3_north_south.yaml
   - name: shaker-l3-east-west
     enabled: true
     type: shaker
     density: 1
     compute: 1
     time: 60
-    file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml
+    file: dense_l3_east_west.yaml
 # Rally
   - name: authenticate
     enabled: true

browbeat/path.py

@@ -18,7 +18,6 @@ _logger = logging.getLogger('browbeat.path')
 
 # Browbeat's project modules path, typically /home/stack/browbeat/browbeat
 browbeat_project_path = os.path.dirname(os.path.realpath(__file__))
-
 # Path to Browbeat, typically /home/stack/browbeat
 browbeat_path = os.path.abspath(os.path.join(browbeat_project_path, os.pardir))
 
@@ -69,3 +68,10 @@ def get_workload_venv(workload, path_activate):
         else:
             _logger.debug("{} not installed in {}".format(workload, workload_venv_path))
     _logger.error('{} does not appear to be installed correctly'.format(workload))
+
+def get_python_site_package(workload):
+    venv = get_workload_venv(workload, False)
+    for root, dirs, files in os.walk(venv):
+        for dir in dirs:
+            if workload in dir:
+                return(os.path.abspath(os.path.join(root,dir)))

browbeat/workloads/shaker.py

@@ -23,6 +23,7 @@ import browbeat.tools
 from browbeat import elastic
 from browbeat import grafana
 from browbeat.path import get_overcloudrc
+from browbeat.path import get_python_site_package
 from browbeat.path import get_workload_venv
 from browbeat.path import results_path
 from browbeat.workloads import base
@@ -374,10 +375,7 @@ class Shaker(base.WorkloadBase):
         self.logger.info("Starting Shaker workloads")
         time_stamp = datetime.datetime.utcnow().strftime("%Y%m%d-%H%M%S")
         self.logger.debug("Time Stamp (Prefix): {}".format(time_stamp))
-
-        venv = get_workload_venv('shaker', False)
         self.shaker_checks()
-
         self.update_total_scenarios()
         shaker_uuid = uuid.uuid4()
         es_ts = datetime.datetime.utcnow()
@@ -396,7 +394,8 @@ class Shaker(base.WorkloadBase):
         for run in rerun_range:
             self.logger.info("Scenario: {}".format(workload['name']))
             self.logger.info("Run: {}".format(run))
-            fname = os.path.join(venv, workload['file'])
+            shaker_package_path = get_python_site_package('shaker')
+            fname = os.path.join(shaker_package_path, "scenarios/openstack", workload['file'])
             self.set_scenario(workload, fname, 60)
             self.logger.debug("Set Scenario File: {}".format(fname))
             result_dir = self.tools.create_results_dir(

conf/quickstart.yml

@@ -53,6 +53,7 @@ shaker:
   shaker_region: regionOne
   external_host: 2.2.2.2
 
+# file for shaker should be one of https://opendev.org/performa/shaker/src/branch/master/shaker/scenarios/openstack
 workloads:
   - name: ping-m1-tiny-centos
     enabled: true
@@ -65,7 +66,7 @@ workloads:
     compute: 1
     progression: linear
     time: 60
-    file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml
+    file: dense_l2.yaml
   - name: quickstart-rally
     enabled: true
     type: rally