Add a backoff for failed builds

If a build has a systematic failure, we currently just let nodepool
run, looping builds until we hit the overall job timeout (1.5 hours).
This adds a count of output log files; if we see three failed builds,
then assume the problem won't get better and fail early.

Change-Id: Id7e163b4937dd57cc8afbf72ed795f73b46a05b1

tools/check_devstack_plugin.sh

@@ -47,12 +47,19 @@ function sshintonode {
 }
 
 function waitforimage {
-    name=$1
-    state='ready'
+    local name=$1
+    local state='ready'
+    local builds
 
     while ! $NODEPOOL image-list | grep $name | grep $state; do
         $NODEPOOL image-list > ${LOGDIR}/nodepool-image-list.txt
         $NODEPOOL list > ${LOGDIR}/nodepool-list.txt
+
+        builds=$(ls -l /var/log/nodepool/builds/ | grep $name | wc -l)
+        if [[ ${builds} -ge 4 ]]; then
+            echo "*** Build of $name failed at least 3 times, aborting"
+            exit 1
+        fi
         sleep 10
     done
 }