Make results and log directory path using browbeat-config

* if results and log path not found, create it.
* Also resolve $FOO in browbeat config paths -jkilpatr

Change-Id: Ie5ec32386ca0d6db9177d9a3a55387b5b1e88a69

ansible/oooq/roles/pre-install-setup/tasks/main.yml

@@ -44,3 +44,6 @@
 - name: Install Ansible
   pip: name=ansible state=present
   become: true
+
+- name: Make results directory
+  file: "path={{ ansible_env.HOME }}/browbeat/results state=directory"

ansible/oooq/roles/pre-install-setup/templates/browbeat-api-ci.yaml.j2

@@ -1,7 +1,8 @@
 browbeat:
-  results : results/
+  results : "$HOME/browbeat/results"
   rerun: 3
   cloud_name: {{ browbeat_cloud_name }}
+  log : "$HOME/browbeat/log"
 elasticsearch:
   enabled: {{ elastic_enabled_template }}
   host: {{ elastic_host_template }}

ansible/oooq/roles/pre-install-setup/templates/browbeat-basic.yaml.j2

@@ -1,9 +1,10 @@
 # Tests to be compleated for the install-and-check.sh script minimal and short workloads are performed
 #  to confirm functionality.
 browbeat:
-  results : results/
+  results : "$HOME/browbeat/results"
   rerun: 1
   cloud_name: {{ browbeat_cloud_name }}
+  log : "$HOME/browbeat/log"
 elasticsearch:
   enabled: {{ elastic_enabled_template }}
   host: {{ elastic_host_template }}

browbeat-complete.yaml

@@ -1,8 +1,9 @@
 # Complete set of Stress Tests, this can take a long time (day(s))
 browbeat:
-  results : results/
+  results : "$HOME/.browbeat/results/"
   rerun: 3
   cloud_name: openstack
+  log : "$HOME/.browbeat/log/"
 elasticsearch:
   enabled: false
   host: 1.1.1.1

browbeat-config.yaml

@@ -1,8 +1,9 @@
 # Basic set of initial stress tests to test overcloud before running complete set of benchmarks.
 browbeat:
-  results : results/
+  results : "$HOME/.browbeat/results"
   rerun: 1
   cloud_name: openstack
+  log: "$HOME/.browbeat/log"
 elasticsearch:
   enabled: true
   host: 1.1.1.1

browbeat.py

@@ -26,10 +26,10 @@ import os
 
 _workload_opts = ['perfkit', 'rally', 'shaker']
 _config_file = 'browbeat-config.yaml'
-debug_log_file = 'log/debug.log'
 
 def main():
     tools = lib.Tools.Tools()
+
     parser = argparse.ArgumentParser(
         description="Browbeat Performance and Scale testing for Openstack")
     parser.add_argument(
@@ -47,7 +47,14 @@ def main():
     _logger.setLevel(logging.DEBUG)
     _formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)7s - %(message)s')
     _formatter.converter = time.gmtime
-    _dbg_file = logging.FileHandler(debug_log_file)
+
+    # Load Browbeat yaml config file:
+    _config = tools._load_config(_cli_args.setup)
+    debug_log_path = tools.sub_env_vars(_config['browbeat']['log'])
+    # create log dir during run time
+    if not os.path.exists(debug_log_path):
+        os.makedirs(debug_log_path)
+    _dbg_file = logging.FileHandler(os.path.join(debug_log_path, 'debug.log'))
     _dbg_file.setLevel(logging.DEBUG)
     _dbg_file.setFormatter(_formatter)
     _ch = logging.StreamHandler()
@@ -61,10 +68,7 @@ def main():
 
     _logger.debug("CLI Args: {}".format(_cli_args))
 
-    # Load Browbeat yaml config file:
+        # Default to all workloads
-    _config = tools._load_config(_cli_args.setup)
-
-    # Default to all workloads
     if _cli_args.workloads == []:
         _cli_args.workloads.append('all')
 
@@ -102,8 +106,10 @@ def main():
                                                                      _cli_args.setup))
             else:
                 _logger.error("{} is missing in {}".format(wkld_provider, _cli_args.setup))
-        result_dir = _config['browbeat']['results']
+        result_dir = tools.sub_env_vars(_config['browbeat']['results'])
         lib.WorkloadBase.WorkloadBase.print_report(result_dir, time_stamp)
+        if not os.path.exists(result_dir):
+            os.makedirs(result_dir)
         _logger.info("Saved browbeat result summary to {}".format(
             os.path.join(result_dir,time_stamp + '.' + 'report')))
         lib.WorkloadBase.WorkloadBase.print_summary()

ci-scripts/config/browbeat-ci.yaml

@@ -1,9 +1,10 @@
 # Tests to be compleated for the install-and-check.sh script minimal and short workloads are performed
 #  to confirm functionality.
 browbeat:
-  results : results/
+  results : "$HOME/.browbeat/results"
   rerun: 1
   cloud_name: openstack
+  log: "$HOME/.browbeat/log"
 elasticsearch:
   enabled: false
   host: 1.1.1.1

lib/Tools.py

@@ -126,3 +126,20 @@ class Tools(object):
         else:
             self.logger.info("Metadata about cloud has been gathered")
             return True
+
+    # Takes a string with $FOO and returns a string
+    # with those variables replaced with their env paths.
+    def sub_env_vars(self, path):
+        path_list = path.split('/')
+        new_path = []
+        for item in path_list:
+            if item.startswith('$'):
+                var = item.translate(None, '$')
+                replacement = os.environ[var]
+                if replacement == None:
+                    new_path.extend(item + "/")
+                else:
+                    new_path.extend(replacement + "/")
+            else:
+                new_path.extend(item + "/")
+        return ''.join(new_path)

lib/validate.yaml

@@ -15,6 +15,9 @@ mapping:
       cloud_name:
         type: str
         required: True
+      log:
+        type: str
+        required: True
 
   elasticsearch:
     required: True

log/.gitignore

@@ -1,4 +0,0 @@
-# Ignore everything in this directory
-*
-# Except this file
-!.gitignore

results/.gitignore

@@ -1,4 +0,0 @@
-# Ignore everything in this directory
-*
-# Except this file
-!.gitignore