From 2357ad47439db76108363bc0e020858561f0d97f Mon Sep 17 00:00:00 2001
From: Sai Sindhur Malleni <smalleni@redhat.com>
Date: Wed, 7 Sep 2016 18:41:51 -0400
Subject: [PATCH] Gather Metadata in main browbeat run

This commit adds logic to run Metadata at the beginning of every browbeat
run through browbeat.py. Also some imports have been fixed.

Co-Authored-By: Joe Talerico <jtaleric@redhat.com>

Change-Id: Ibc13a64710209b25a755f606ea7fddc80232cbc4
---
 ansible/generate_tripleo_hostfile.sh          |  7 ++--
 ansible/install/roles/browbeat/tasks/main.yml |  1 -
 browbeat-complete.yaml                        |  2 ++
 browbeat-config.yaml                          |  2 ++
 browbeat.py                                   | 34 +++++++++++++------
 ci-scripts/config/browbeat-ci.yaml            |  2 ++
 conf/browbeat-glance-complete.yaml            |  2 ++
 conf/browbeat-keystone-complete.yaml          |  2 ++
 conf/browbeat-keystone-minimal.yaml           |  2 ++
 conf/browbeat-perfkit-complete.yaml           |  2 ++
 conf/ceilometer-minimal.yaml                  |  2 ++
 conf/ceilometer-persist-response-timing.yaml  |  2 ++
 conf/telemetry-nova-persist-instances.yaml    |  2 ++
 lib/validate.yaml                             |  7 ++++
 14 files changed, 54 insertions(+), 15 deletions(-)

diff --git a/ansible/generate_tripleo_hostfile.sh b/ansible/generate_tripleo_hostfile.sh
index 9a8864150..05cfd258a 100755
--- a/ansible/generate_tripleo_hostfile.sh
+++ b/ansible/generate_tripleo_hostfile.sh
@@ -6,7 +6,8 @@ if [ ! $# -ge 1 ]; then
 fi
 tripleo_ip_address=$1
 ansible_inventory_file='hosts'
-ssh_config_file='ssh-config'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+ssh_config_file=${DIR}'/ssh-config'
 
 # "Hackish" copy ssh key to self if we are on directly on the undercloud machine:
 if [[ "${tripleo_ip_address}" == "localhost" ]]; then
@@ -107,9 +108,9 @@ for line in $nodes; do
  fi
  echo "" | tee -a ${ssh_config_file}
  echo "Host ${host}" | tee -a ${ssh_config_file}
- echo "    ProxyCommand ssh -F ssh-config -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ConnectTimeout=60 -i ~/.ssh/id_rsa stack@${tripleo_ip_address} -W ${IP}:22" | tee -a ${ssh_config_file}
+ echo "    ProxyCommand ssh -F ${ssh_config_file} -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ConnectTimeout=60 -i ~/.ssh/id_rsa stack@${tripleo_ip_address} -W ${IP}:22" | tee -a ${ssh_config_file}
  echo "    User heat-admin" | tee -a ${ssh_config_file}
- echo "    IdentityFile heat-admin-id_rsa" | tee -a ${ssh_config_file}
+ echo "    IdentityFile ${DIR}/heat-admin-id_rsa" | tee -a ${ssh_config_file}
  echo "    StrictHostKeyChecking no" | tee -a ${ssh_config_file}
  echo "    UserKnownHostsFile=/dev/null" | tee -a ${ssh_config_file}
 done
diff --git a/ansible/install/roles/browbeat/tasks/main.yml b/ansible/install/roles/browbeat/tasks/main.yml
index 0e7498e1f..e88f451cc 100644
--- a/ansible/install/roles/browbeat/tasks/main.yml
+++ b/ansible/install/roles/browbeat/tasks/main.yml
@@ -89,7 +89,6 @@
   command: mv {{ home_dir }}/{{item}} {{ browbeat_path }}/ansible/{{item}}
   with_items:
     - hosts
-    - ssh-config
     - heat-admin-id_rsa
 
 - name: Install requirements.txt into browbeat-venv
diff --git a/browbeat-complete.yaml b/browbeat-complete.yaml
index 7c0c9879f..ce0a04c69 100644
--- a/browbeat-complete.yaml
+++ b/browbeat-complete.yaml
@@ -15,6 +15,7 @@ elasticsearch:
     - name: software-metadata
       file: metadata/software-metadata.json
 ansible:
+  ssh_config: ansible/ssh-config
   hosts: ansible/hosts
   adjust:
     keystone_token: ansible/browbeat/adjustment-keystone-token.yml
@@ -22,6 +23,7 @@ ansible:
     nova_db: ansible/browbeat/adjustment-db.yml
     workers: ansible/browbeat/adjustment-workers.yml
   grafana_snapshot: ansible/browbeat/snapshot-general-performance-dashboard.yml
+  metadata: ansible/gather/site.yml
 connmon:
   enabled: false
   sudo: true
diff --git a/browbeat-config.yaml b/browbeat-config.yaml
index 53e1b940b..5fc94ebb2 100644
--- a/browbeat-config.yaml
+++ b/browbeat-config.yaml
@@ -15,6 +15,7 @@ elasticsearch:
     - name: software-metadata
       file: metadata/software-metadata.json
 ansible:
+  ssh_config: ansible/ssh-config
   hosts: ansible/hosts
   adjust:
     keystone_token: ansible/browbeat/adjustment-keystone-token.yml
@@ -22,6 +23,7 @@ ansible:
     nova_db: ansible/browbeat/adjustment-db.yml
     workers: ansible/browbeat/adjustment-workers.yml
   grafana_snapshot: ansible/browbeat/snapshot-general-performance-dashboard.yml
+  metadata: ansible/gather/site.yml
 connmon:
   enabled: false
   sudo: true
diff --git a/browbeat.py b/browbeat.py
index bbb22badc..d04b261a1 100755
--- a/browbeat.py
+++ b/browbeat.py
@@ -12,10 +12,11 @@
 #   limitations under the License.
 
 from lib.Elastic import browbeat_uuid
-from lib import PerfKit
-from lib import Rally
-from lib import Shaker
-from lib import WorkloadBase
+import lib.PerfKit
+import lib.Rally
+import lib.Shaker
+import lib.WorkloadBase
+import lib.Tools
 import argparse
 import logging
 import sys
@@ -56,13 +57,13 @@ def validate_yaml(config, _logger):
 def _run_workload_provider(provider, config):
     _logger = logging.getLogger('browbeat')
     if provider == "perfkit":
-        perfkit = PerfKit.PerfKit(config)
+        perfkit = lib.PerfKit.PerfKit(config)
         perfkit.start_workloads()
     elif provider == "rally":
-        rally = Rally.Rally(config)
+        rally = lib.Rally.Rally(config)
         rally.start_workloads()
     elif provider == "shaker":
-        shaker = Shaker.Shaker(config)
+        shaker = lib.Shaker.Shaker(config)
         shaker.run_shaker()
     else:
         _logger.error("Unknown workload provider: {}".format(provider))
@@ -133,8 +134,19 @@ def main():
             metadata_exists = check_metadata(_config, _logger)
             if not metadata_exists:
                 _logger.error("Elasticsearch has been enabled but"
-                              " metadata files do not exist, exiting")
-                sys.exit(1)
+                              " metadata files do not exist")
+                _logger.info("Gathering Metadata")
+                os.putenv("ANSIBLE_SSH_ARGS"," -F {}".format(_config['ansible']['ssh_config']))
+                tools = lib.Tools.Tools(_config)
+                ansible_cmd = \
+                    'ansible-playbook -i {} {}' \
+                    .format(_config['ansible']['hosts'], _config['ansible']['metadata'])
+                tools.run_cmd(ansible_cmd)
+                if not check_metadata(_config, _logger):
+                    _logger.warning("Metadata could not be gathered")
+                    exit(1)
+                else:
+                    _logger.info("Metadata about cloud has been gathered")
         _logger.info("Running workload(s): {}".format(','.join(_cli_args.workloads)))
         for wkld_provider in _cli_args.workloads:
             if wkld_provider in _config:
@@ -146,10 +158,10 @@ def main():
             else:
                 _logger.error("{} is missing in {}".format(wkld_provider, _cli_args.setup))
         result_dir = _config['browbeat']['results']
-        WorkloadBase.WorkloadBase.print_report(result_dir, time_stamp)
+        lib.WorkloadBase.WorkloadBase.print_report(result_dir, time_stamp)
         _logger.info("Saved browbeat result summary to {}".format(
             os.path.join(result_dir,time_stamp + '.' + 'report')))
-        WorkloadBase.WorkloadBase.print_summary()
+        lib.WorkloadBase.WorkloadBase.print_summary()
         _logger.info("Browbeat Finished, UUID: {}".format(browbeat_uuid))
 
 if __name__ == '__main__':
diff --git a/ci-scripts/config/browbeat-ci.yaml b/ci-scripts/config/browbeat-ci.yaml
index 8569b79fb..d80a4e5f9 100644
--- a/ci-scripts/config/browbeat-ci.yaml
+++ b/ci-scripts/config/browbeat-ci.yaml
@@ -16,6 +16,7 @@ elasticsearch:
     - name: software-metadata
       file: metadata/software-metadata.json
 ansible:
+  ssh_config: ansible/ssh-config
   hosts: ansible/hosts
   adjust:
     keystone_token: ansible/browbeat/adjustment-keystone-token.yml
@@ -23,6 +24,7 @@ ansible:
     nova_db: ansible/browbeat/adjustment-db.yml
     workers: ansible/browbeat/adjustment-workers.yml
   grafana_snapshot: ansible/browbeat/snapshot-general-performance-dashboard.yml
+  metadata: ansible/gather/site.yml
 connmon:
   enabled: false
   sudo: true
diff --git a/conf/browbeat-glance-complete.yaml b/conf/browbeat-glance-complete.yaml
index a6b0ce08e..2543bbbad 100644
--- a/conf/browbeat-glance-complete.yaml
+++ b/conf/browbeat-glance-complete.yaml
@@ -15,6 +15,7 @@ elasticsearch:
     - name: software-metadata
       file: metadata/software-metadata.json
 ansible:
+  ssh_config: ansible/ssh-config
   hosts: ansible/hosts
   adjust:
     keystone_token: ansible/browbeat/adjustment-keystone-token.yml
@@ -22,6 +23,7 @@ ansible:
     nova_db: ansible/browbeat/adjustment-db.yml
     workers: ansible/browbeat/adjustment-workers.yml
   grafana_snapshot: ansible/browbeat/snapshot-general-performance-dashboard.yml
+  metadata: ansible/gather/site.yml
 connmon:
   enabled: false
   sudo: true
diff --git a/conf/browbeat-keystone-complete.yaml b/conf/browbeat-keystone-complete.yaml
index 07d1e701e..1c5f42089 100644
--- a/conf/browbeat-keystone-complete.yaml
+++ b/conf/browbeat-keystone-complete.yaml
@@ -16,6 +16,7 @@ elasticsearch:
     - name: software-metadata
       file: metadata/software-metadata.json
 ansible:
+  ssh_config: ansible/ssh-config
   hosts: ansible/hosts
   adjust:
     keystone_token: ansible/browbeat/adjustment-keystone-token.yml
@@ -23,6 +24,7 @@ ansible:
     nova_db: ansible/browbeat/adjustment-db.yml
     workers: ansible/browbeat/adjustment-workers.yml
   grafana_snapshot: ansible/browbeat/snapshot-general-performance-dashboard.yml
+  metadata: ansible/gather/site.yml
 connmon:
   enabled: false
   sudo: true
diff --git a/conf/browbeat-keystone-minimal.yaml b/conf/browbeat-keystone-minimal.yaml
index 1a1443c0a..93f7881a9 100644
--- a/conf/browbeat-keystone-minimal.yaml
+++ b/conf/browbeat-keystone-minimal.yaml
@@ -17,6 +17,7 @@ elasticsearch:
       file: metadata/software-metadata.json
 
 ansible:
+  ssh_config: ansible/ssh-config
   hosts: ansible/hosts
   adjust:
     keystone_token: ansible/browbeat/adjustment-keystone-token.yml
@@ -24,6 +25,7 @@ ansible:
     nova_db: ansible/browbeat/adjustment-db.yml
     workers: ansible/browbeat/adjustment-workers.yml
   grafana_snapshot: ansible/browbeat/snapshot-general-performance-dashboard.yml
+  metadata: ansible/gather/site.yml
 connmon:
   enabled: false
   sudo: true
diff --git a/conf/browbeat-perfkit-complete.yaml b/conf/browbeat-perfkit-complete.yaml
index fd441ab83..95a5a1444 100644
--- a/conf/browbeat-perfkit-complete.yaml
+++ b/conf/browbeat-perfkit-complete.yaml
@@ -15,6 +15,7 @@ elasticsearch:
     - name: software-metadata
       file: metadata/software-metadata.json
 ansible:
+  ssh_config: ansible/ssh-config
   hosts: ansible/hosts
   adjust:
     keystone_token: ansible/browbeat/adjustment-keystone-token.yml
@@ -22,6 +23,7 @@ ansible:
     nova_db: ansible/browbeat/adjustment-db.yml
     workers: ansible/browbeat/adjustment-workers.yml
   grafana_snapshot: ansible/browbeat/snapshot-general-performance-dashboard.yml
+  metadata: ansible/gather/site.yml
 connmon:
   enabled: false
   sudo: true
diff --git a/conf/ceilometer-minimal.yaml b/conf/ceilometer-minimal.yaml
index cc7e548c6..ef88251d9 100644
--- a/conf/ceilometer-minimal.yaml
+++ b/conf/ceilometer-minimal.yaml
@@ -15,6 +15,7 @@ elasticsearch:
     - name: software-metadata
       file: metadata/software-metadata.json
 ansible:
+  ssh_config: ansible/ssh-config
   hosts: ansible/hosts
   adjust:
     keystone_token: ansible/browbeat/adjustment-keystone-token.yml
@@ -22,6 +23,7 @@ ansible:
     nova_db: ansible/browbeat/adjustment-db.yml
     workers: ansible/browbeat/adjustment-workers.yml
   grafana_snapshot: ansible/browbeat/snapshot-general-performance-dashboard.yml
+  metadata: ansible/gather/site.yml
 connmon:
   enabled: false
   sudo: true
diff --git a/conf/ceilometer-persist-response-timing.yaml b/conf/ceilometer-persist-response-timing.yaml
index f6d9fb2cf..df641ff0a 100644
--- a/conf/ceilometer-persist-response-timing.yaml
+++ b/conf/ceilometer-persist-response-timing.yaml
@@ -18,6 +18,7 @@ elasticsearch:
     - name: software-metadata
       file: metadata/software-metadata.json
 ansible:
+  ssh_config: ansible/ssh-config
   hosts: ansible/hosts
   adjust:
     keystone_token: ansible/browbeat/adjustment-keystone-token.yml
@@ -25,6 +26,7 @@ ansible:
     nova_db: ansible/browbeat/adjustment-db.yml
     workers: ansible/browbeat/adjustment-workers.yml
   grafana_snapshot: ansible/browbeat/snapshot-general-performance-dashboard.yml
+  metadata: ansible/gather/site.yml
 connmon:
   enabled: false
   sudo: true
diff --git a/conf/telemetry-nova-persist-instances.yaml b/conf/telemetry-nova-persist-instances.yaml
index c5d7bbf6b..dc7924724 100644
--- a/conf/telemetry-nova-persist-instances.yaml
+++ b/conf/telemetry-nova-persist-instances.yaml
@@ -18,6 +18,7 @@ elasticsearch:
     - name: software-metadata
       file: metadata/software-metadata.json
 ansible:
+  ssh_config: ansible/ssh-config
   hosts: ansible/hosts
   adjust:
     keystone_token: ansible/browbeat/adjustment-keystone-token.yml
@@ -25,6 +26,7 @@ ansible:
     nova_db: ansible/browbeat/adjustment-db.yml
     workers: ansible/browbeat/adjustment-workers.yml
   grafana_snapshot: ansible/browbeat/snapshot-general-performance-dashboard.yml
+  metadata: ansible/gather/site.yml
 connmon:
   enabled: false
   sudo: true
diff --git a/lib/validate.yaml b/lib/validate.yaml
index 15e99e9c1..f4f3c3136 100644
--- a/lib/validate.yaml
+++ b/lib/validate.yaml
@@ -50,8 +50,12 @@ mapping:
     type: map
     allowempty: True
     mapping:
+      ssh_config:
+        type: str
+        required: True
       hosts:
         type: str
+        required: True
       adjust:
         type: map
         mapping:
@@ -68,6 +72,9 @@ mapping:
         required: True
       shaker_build:
         type: str
+      metadata:
+        type: str
+        required: True
 
   connmon:
     type: map