From 1b1cc8c1d491b1469c7082d28d3e4a3a8a4a4fd9 Mon Sep 17 00:00:00 2001
From: Ian Wienand <iwienand@redhat.com>
Date: Tue, 1 Mar 2016 13:14:39 +1100
Subject: [PATCH] yum_install_package: fix errexit and retry

Since I93e9f312a94aeb086925e069a83ec1d3d3419423 yum_install isn't safe
under errexit.  This means it really only works when called by
tools/install_prereqs.sh because for some reason, we don't set that
there.

However, there is a problem with the retry logic when detecting failed
installs.  A failed package install should stop further progress, but
with the current retry logic it just goes ahead and retries the
installation, which then incorrectly passes.  You can see this
happening in a test like [1].

In our detection scripts, make a failed package or missing packages
exit with error-code 2, and "die" when we see this to correctly stop.

[1] http://logs.openstack.org/81/285881/1/check/gate-tempest-dsvm-platform-fedora23-nv/a83be30/logs/devstacklog.txt.gz

Change-Id: I4ea5515fa8e82a66aefa3ec3a48b823b645274f7
---
 functions-common | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/functions-common b/functions-common
index 2b23402b54..72ac3fd96a 100644
--- a/functions-common
+++ b/functions-common
@@ -1302,13 +1302,14 @@ function yum_install {
 
     time_start "yum_install"
 
-    # Warning: this would not work if yum output message
-    # have been translated to another language
+    # - We run with LC_ALL=C so string matching *should* be OK
+    # - Exit 1 if the failure might get better with a retry.
+    # - Exit 2 if it is fatal.
     parse_yum_result='             \
         BEGIN { result=0 }         \
         /^YUM_FAILED/ { exit $2 }  \
-        /^No package/ { result=1 } \
-        /^Failed:/    { result=1 } \
+        /^No package/ { result=2 } \
+        /^Failed:/    { result=2 } \
         //{ print }                \
         END { exit result }'
 
@@ -1316,15 +1317,21 @@ function yum_install {
     # missing or failed packages are OK.
     # See https://bugzilla.redhat.com/show_bug.cgi?id=965567
     (sudo_with_proxies "${YUM:-yum}" install -y "$@" 2>&1 || echo YUM_FAILED $?) \
-        | awk "$parse_yum_result"
-    result=$?
-
-    if [ "$result" != 0 ]; then
-        echo $LINENO "${YUM:-yum}" install failure: $result
-    fi
+        | awk "$parse_yum_result" && result=$? || result=$?
 
     time_stop "yum_install"
 
+    # if we return 1, then the wrapper functions will run an update
+    # and try installing the package again as a defense against bad
+    # mirrors.  This can hide failures, especially when we have
+    # packages that are in the "Failed:" section because their rpm
+    # install scripts failed to run correctly (in this case, the
+    # package looks installed, so when the retry happens we just think
+    # the package is OK, and incorrectly continue on).
+    if [ "$result" == 2 ]; then
+        die "Detected fatal package install failure"
+    fi
+
     return "$result"
 }