airship/vino
Code Issues Proposed changes

Fix race condition in deployment script

Browse Source 
This fixes a(nother) race condition in the gate, where `kubectl wait`
was getting called before the objects being waited on existed, which
causes it to fail [0].

It also moves to using `kubectl rollout status` for daemonsets/deployments,
for a bit more robustness.

[0]: https://github.com/kubernetes/kubernetes/issues/83242

Signed-off-by: Sean Eagan <seaneagan1@gmail.com>
Change-Id: I87014a3bae05ad35f063c1082893a3075a05e3ee
This commit is contained in:
Sean Eagan 2021-02-10 13:23:43 -06:00
parent bccf6d68c5
commit 817cbfc967
2 changed files with 22 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
tools/deployment/deploy-vino.sh
View File

@ -7,12 +7,12 @@ make deploy
kubectl get po -A kubectl get po -A
#Wait for vino controller manager Pod. #Wait for vino controller manager Pod.
count=0 count=0
until [[ $(kubectl -n vino-system get pod -l control-plane=controller-manager 2>/dev/null) ]]; do until [[ $(kubectl -n vino-system get deployment -l control-plane=controller-manager 2>/dev/null) ]]; do
count=$((count + 1)) count=$((count + 1))
if [[ ${count} -eq "120" ]]; then if [[ ${count} -eq "120" ]]; then
echo ' Timed out waiting for vino controller manager pod to exist' echo ' Timed out waiting for vino controller manager deployment to exist'
return 1 return 1
fi fi
sleep 2 sleep 2
done done
kubectl wait -n vino-system pod -l control-plane=controller-manager --for=condition=ready --timeout=240s kubectl -n vino-system rollout status deployment vino-controller-manager --timeout=240s

20
tools/deployment/test-cr.sh
View File

@ -15,11 +15,29 @@ function vinoDebugInfo () {
kubectl apply -f config/samples/vino_cr.yaml kubectl apply -f config/samples/vino_cr.yaml
# Remove logs collection from here, when we will have zuul collect logs job # Remove logs collection from here, when we will have zuul collect logs job
until [[ $(kubectl get vino vino-test-cr 2>/dev/null) ]]; do
count=$((count + 1))
if [[ ${count} -eq "30" ]]; then
echo ' Timed out waiting for vino test cr to exist'
vinoDebugInfo
return 1
fi
sleep 2
done
if ! kubectl wait --for=condition=Ready vino vino-test-cr --timeout=180s; then if ! kubectl wait --for=condition=Ready vino vino-test-cr --timeout=180s; then
vinoDebugInfo vinoDebugInfo
fi fi
# no need to collect logs on fail, since they are already collected before # no need to collect logs on fail, since they are already collected before
if ! kubectl wait --for=condition=Ready pods -l 'vino-role=vino-builder' --timeout=5s; then until [[ $(kubectl get ds vino-test-cr 2>/dev/null) ]]; do
count=$((count + 1))
if [[ ${count} -eq "30" ]]; then
echo ' Timed out waiting for vino builder daemonset to exist'
vinoDebugInfo
return 1
fi
sleep 2
done
if ! kubectl rollout status ds vino-test-cr --timeout=10s; then
vinoDebugInfo vinoDebugInfo
fi fi