Fix automatic log copying in Zuul runs

Key off an environment variable, "ZUUL_PROJECT", instead of the
existence of /etc/nodepool, when determining whether to do a log
copy into the workspace at the end of the job.

Change-Id: I3037249ff847cc805ac83f71e7df5943efe7eb25

scripts/gate-check-commit.sh

@@ -74,9 +74,6 @@ info_block "Checking for required libraries." 2> /dev/null || source "${OSA_CLON
 
 ## Main ----------------------------------------------------------------------
 
-# Set gate job exit traps, this is run regardless of exit state when the job finishes.
-trap gate_job_exit_tasks EXIT
-
 # Log some data about the instance and the rest of the system
 log_instance_info
 

scripts/scripts-library.sh

@@ -22,13 +22,14 @@ ANSIBLE_PARAMETERS=${ANSIBLE_PARAMETERS:-""}
 STARTTIME="${STARTTIME:-$(date +%s)}"
 COMMAND_LOGS=${COMMAND_LOGS:-"/openstack/log/ansible_cmd_logs"}
 
+ZUUL_PROJECT="${ZUUL_PROJECT:-}"
 GATE_EXIT_LOG_COPY="${GATE_EXIT_LOG_COPY:-false}"
 GATE_EXIT_LOG_GZIP="${GATE_EXIT_LOG_GZIP:-true}"
 GATE_EXIT_RUN_ARA="${GATE_EXIT_RUN_ARA:-true}"
 GATE_EXIT_RUN_DSTAT="${GATE_EXIT_RUN_DSTAT:-true}"
 # If this is a gate node from OpenStack-Infra Store all logs into the
 #  execution directory after gate run.
-if [[ -d "/etc/nodepool" ]]; then
+if [[ -n "$ZUUL_PROJECT" ]]; then
   GATE_EXIT_LOG_COPY=true
 fi
 
@@ -137,9 +138,9 @@ function gate_job_exit_tasks {
     fi
     GATE_LOG_DIR="${OSA_CLONE_DIR:-$(dirname $0)/..}/logs"
     mkdir -p "${GATE_LOG_DIR}/host" "${GATE_LOG_DIR}/openstack"
-    RSYNC_CMD="rsync --archive --safe-links --ignore-errors --quiet --no-perms --no-owner --no-group"
+    RSYNC_OPTS="--archive --safe-links --ignore-errors --quiet --no-perms --no-owner --no-group"
-    ${RSYNC_CMD} /var/log/ "${GATE_LOG_DIR}/host" || true
+    rsync $RSYNC_OPTS /var/log/ "${GATE_LOG_DIR}/host" || true
-    ${RSYNC_CMD} /openstack/log/ "${GATE_LOG_DIR}/openstack" || true
+    rsync $RSYNC_OPTS /openstack/log/ "${GATE_LOG_DIR}/openstack" || true
     # Rename all files gathered to have a .txt suffix so that the compressed
     # files are viewable via a web browser in OpenStack-CI.
     # except tempest results testrepository.subunit and testr_results.html
@@ -167,7 +168,8 @@ function gate_job_exit_tasks {
     fi
     # Ensure that the files are readable by all users, including the non-root
     # OpenStack-CI jenkins user.
-    chmod -R 0777 "${GATE_LOG_DIR}"
+    chmod -R ugo+rX "${GATE_LOG_DIR}"
+    chown -R $(whoami) "${GATE_LOG_DIR}"
   fi
 }
 
@@ -187,7 +189,10 @@ function run_dstat {
         ;;
   esac
 
-  dstat -tcmsdn --top-cpu --top-mem --top-bio --nocolor --output /openstack/log/instance-info/dstat.csv 3 > /openstack/log/instance-info/dstat.log&
+  # https://stackoverflow.com/a/20338327 executing in ()& decouples the dstat
+  # process from scripts-library to prevent hung builds if dstat fails to exit
+  # for any reason.
+  (dstat -tcmsdn --top-cpu --top-mem --top-bio --nocolor --output /openstack/log/instance-info/dstat.csv 3 > /openstack/log/instance-info/dstat.log)&
 }
 
 function generate_dstat_charts {

scripts/test-log-collect.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Copyright 2017, Logan Vig <logan2211@gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## Shell Opts ----------------------------------------------------------------
+set -e -u -x
+
+# Store the clone repo root location
+export OSA_CLONE_DIR="$(readlink -f $(dirname ${0})/..)"
+
+## Functions -----------------------------------------------------------------
+info_block "Checking for required libraries." 2> /dev/null || source "${OSA_CLONE_DIR}/scripts/scripts-library.sh"
+
+## Main ----------------------------------------------------------------------
+gate_job_exit_tasks

zuul.d/jobs.yaml

@@ -21,6 +21,7 @@
       Uses the gate-check-commit.sh script, running a default
       aio deploy.
     run: zuul.d/playbooks/run.yml
+    post-run: zuul.d/playbooks/post.yml
     timeout: 7200
     irrelevant-files:
       - ^\.git.*

zuul.d/playbooks/post.yml

@@ -0,0 +1,32 @@
+---
+# Copyright 2017, Logan Vig <logan2211@gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: all
+  tasks:
+    - name: Run log collection script
+      command: scripts/test-log-collect.sh
+      become: yes
+      become_user: root
+      args:
+        chdir: "src/{{ zuul.project.canonical_name }}"
+      environment:
+        # ZUUL_PROJECT is used by the log collection functions to enable
+        # log collection configuration specific to OpenStack CI
+        ZUUL_PROJECT: "{{ zuul.project.short_name }}"
+    - name: Copy logs back to the executor
+      synchronize:
+        src: "src/{{ zuul.project.canonical_name }}/logs"
+        dest: "{{ zuul.executor.log_root }}"
+        mode: pull