From af28cf025ba0ccbad8bca215de767fb97d0cc1c3 Mon Sep 17 00:00:00 2001
From: Logan V <logan2211@gmail.com>
Date: Wed, 27 Sep 2017 14:19:03 -0500
Subject: [PATCH] Run dstat in gate tests

This will enable dstat dumps at 3 second intervals in the background
during gate jobs. In the gate job exit tasks, dstat_graph[1] will be
used to generate an HTML compilation of the dstat data.

[1] https://github.com/Dabz/dstat_graph

Change-Id: I8475098a75528f8260a41f853a243b08296e3855
---
 scripts/gate-check-commit.sh |  7 ++++---
 scripts/scripts-library.sh   | 35 ++++++++++++++++++++++++++++++++++-
 2 files changed, 38 insertions(+), 4 deletions(-)

diff --git a/scripts/gate-check-commit.sh b/scripts/gate-check-commit.sh
index 371ca3426e..a9a4ab14e9 100755
--- a/scripts/gate-check-commit.sh
+++ b/scripts/gate-check-commit.sh
@@ -80,6 +80,10 @@ trap gate_job_exit_tasks EXIT
 # Log some data about the instance and the rest of the system
 log_instance_info
 
+if [ "$GATE_EXIT_RUN_DSTAT" == true ]; then
+  run_dstat
+fi
+
 # Get minimum disk size
 DATA_DISK_MIN_SIZE="$((1024**3 * $(awk '/bootstrap_host_data_disk_min_size/{print $2}' "${OSA_CLONE_DIR}/tests/roles/bootstrap-host/defaults/main.yml") ))"
 
@@ -153,9 +157,6 @@ pushd "${OSA_CLONE_DIR}/tests"
   fi
 popd
 
-# Implement the log directory
-mkdir -p /openstack/log
-
 pushd "${OSA_CLONE_DIR}/playbooks"
   # Disable Ansible color output
   export ANSIBLE_NOCOLOR=1
diff --git a/scripts/scripts-library.sh b/scripts/scripts-library.sh
index 96d37cbee4..bc7fd337ac 100755
--- a/scripts/scripts-library.sh
+++ b/scripts/scripts-library.sh
@@ -25,6 +25,7 @@ COMMAND_LOGS=${COMMAND_LOGS:-"/openstack/log/ansible_cmd_logs"}
 GATE_EXIT_LOG_COPY="${GATE_EXIT_LOG_COPY:-false}"
 GATE_EXIT_LOG_GZIP="${GATE_EXIT_LOG_GZIP:-true}"
 GATE_EXIT_RUN_ARA="${GATE_EXIT_RUN_ARA:-true}"
+GATE_EXIT_RUN_DSTAT="${GATE_EXIT_RUN_DSTAT:-true}"
 # If this is a gate node from OpenStack-Infra Store all logs into the
 #  execution directory after gate run.
 if [[ -d "/etc/nodepool" ]]; then
@@ -126,6 +127,9 @@ function gate_job_exit_tasks {
   # If this is a gate node from OpenStack-Infra Store all logs into the
   #  execution directory after gate run.
   if [ "$GATE_EXIT_LOG_COPY" == true ]; then
+    if [ "$GATE_EXIT_RUN_DSTAT" == true ]; then
+      generate_dstat_charts || true
+    fi
     GATE_LOG_DIR="${OSA_CLONE_DIR:-$(dirname $0)/..}/logs"
     mkdir -p "${GATE_LOG_DIR}/host" "${GATE_LOG_DIR}/openstack"
     rsync --archive --verbose --safe-links --ignore-errors /var/log/ "${GATE_LOG_DIR}/host" || true
@@ -133,7 +137,7 @@ function gate_job_exit_tasks {
     # Rename all files gathered to have a .txt suffix so that the compressed
     # files are viewable via a web browser in OpenStack-CI.
     # except tempest results testrepository.subunit and testr_results.html
-    find "${GATE_LOG_DIR}/" -type f -not -name "testrepository.subunit" -not -name "testr_results.html" -exec mv {} {}.txt \;
+    find "${GATE_LOG_DIR}/" -type f -not -name "testrepository.subunit" -not -name "testr_results.html" -not -name "dstat.html" -exec mv {} {}.txt \;
 
     # Generate the ARA report if enabled
     if [ "$GATE_EXIT_RUN_ARA" == true ]; then
@@ -151,6 +155,35 @@ function gate_job_exit_tasks {
   fi
 }
 
+function run_dstat {
+  case ${DISTRO_ID} in
+    centos|rhel)
+        # Prefer dnf over yum for CentOS.
+        which dnf &>/dev/null && RHT_PKG_MGR='dnf' || RHT_PKG_MGR='yum'
+        $RHT_PKG_MGR -y install dstat
+        ;;
+    ubuntu)
+        apt-get update
+        DEBIAN_FRONTEND=noninteractive apt-get -y install dstat
+        ;;
+    opensuse)
+        zypper -n install -l dstat
+        ;;
+  esac
+
+  dstat -tcmsdn --top-cpu --top-mem --top-bio --nocolor --output /openstack/log/instance-info/dstat.csv 3 > /openstack/log/instance-info/dstat.log&
+}
+
+function generate_dstat_charts {
+  kill $(pgrep dstat)
+  if [[ ! -d /opt/dstat_graph ]]; then
+    git clone https://github.com/Dabz/dstat_graph /opt/dstat_graph
+  fi
+  pushd /opt/dstat_graph
+    /usr/bin/env bash -e ./generate_page.sh /openstack/log/instance-info/dstat.csv >> /openstack/log/instance-info/dstat.html
+  popd
+}
+
 function print_info {
   PROC_NAME="- [ $@ ] -"
   printf "\n%s%s\n" "$PROC_NAME" "${LINE:${#PROC_NAME}}"