From 817cbfc9670bba7e83c7b4d498f64fa2f98bdd00 Mon Sep 17 00:00:00 2001
From: Sean Eagan <seaneagan1@gmail.com>
Date: Wed, 10 Feb 2021 13:23:43 -0600
Subject: [PATCH] Fix race condition in deployment script

This fixes a(nother) race condition in the gate, where `kubectl wait`
was getting called before the objects being waited on existed, which
causes it to fail [0].

It also moves to using `kubectl rollout status` for daemonsets/deployments,
for a bit more robustness.

[0]: https://github.com/kubernetes/kubernetes/issues/83242

Signed-off-by: Sean Eagan <seaneagan1@gmail.com>
Change-Id: I87014a3bae05ad35f063c1082893a3075a05e3ee
---
 tools/deployment/deploy-vino.sh |  6 +++---
 tools/deployment/test-cr.sh     | 20 +++++++++++++++++++-
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/tools/deployment/deploy-vino.sh b/tools/deployment/deploy-vino.sh
index f4c8f0f..389fed8 100755
--- a/tools/deployment/deploy-vino.sh
+++ b/tools/deployment/deploy-vino.sh
@@ -7,12 +7,12 @@ make deploy
 kubectl get po -A
 #Wait for vino controller manager Pod.
 count=0
-until [[ $(kubectl -n vino-system get pod -l control-plane=controller-manager 2>/dev/null) ]]; do
+until [[ $(kubectl -n vino-system get deployment -l control-plane=controller-manager 2>/dev/null) ]]; do
   count=$((count + 1))
   if [[ ${count} -eq "120" ]]; then
-    echo ' Timed out waiting for vino controller manager pod to exist'
+    echo ' Timed out waiting for vino controller manager deployment to exist'
     return 1
   fi
   sleep 2
 done
-kubectl wait -n vino-system pod  -l control-plane=controller-manager --for=condition=ready --timeout=240s
\ No newline at end of file
+kubectl -n vino-system rollout status deployment vino-controller-manager --timeout=240s
\ No newline at end of file
diff --git a/tools/deployment/test-cr.sh b/tools/deployment/test-cr.sh
index ba8a72e..3912aac 100755
--- a/tools/deployment/test-cr.sh
+++ b/tools/deployment/test-cr.sh
@@ -15,11 +15,29 @@ function vinoDebugInfo () {
 kubectl apply -f config/samples/vino_cr.yaml
 
 # Remove logs collection from here, when we will have zuul collect logs job
+until [[ $(kubectl get vino vino-test-cr 2>/dev/null) ]]; do
+  count=$((count + 1))
+  if [[ ${count} -eq "30" ]]; then
+    echo ' Timed out waiting for vino test cr to exist'
+    vinoDebugInfo
+    return 1
+  fi
+  sleep 2
+done
 if ! kubectl wait --for=condition=Ready vino vino-test-cr --timeout=180s; then
     vinoDebugInfo
 fi
 
 # no need to collect logs on fail, since they are already collected before
-if ! kubectl wait --for=condition=Ready pods -l 'vino-role=vino-builder' --timeout=5s; then
+until [[ $(kubectl get ds vino-test-cr 2>/dev/null) ]]; do
+  count=$((count + 1))
+  if [[ ${count} -eq "30" ]]; then
+    echo ' Timed out waiting for vino builder daemonset to exist'
+    vinoDebugInfo
+    return 1
+  fi
+  sleep 2
+done
+if ! kubectl rollout status ds vino-test-cr --timeout=10s; then
     vinoDebugInfo
 fi