From f60481c86a6e8fafb031815272ab8e5df39ac0a6 Mon Sep 17 00:00:00 2001
From: Sergiy Markin <smarkin@mirantis.com>
Date: Tue, 9 Jul 2024 15:32:29 +0000
Subject: [PATCH] [utility] ensure_ondemand_pod_exists improvements

This PS covers the case when ondemand pod does not
exists but the ondemand job is present. In this
case ensure_ondemand_pod_exists() function calls
do_cleanup() before re-creating ondemand job.

Change-Id: Iffd28317f38eb0d3ec47886e790902b12b5e22e3
---
 charts/etcdctl-utility/Chart.yaml             |  2 +-
 .../templates/bin/utility/_dbutils.tpl        | 39 ++++++++-------
 charts/mysqlclient-utility/Chart.yaml         |  2 +-
 .../templates/bin/utility/_dbutils.tpl        | 47 +++++++++++--------
 charts/postgresql-utility/Chart.yaml          |  2 +-
 .../templates/bin/utility/_dbutils.tpl        | 41 +++++++++-------
 6 files changed, 77 insertions(+), 56 deletions(-)

diff --git a/charts/etcdctl-utility/Chart.yaml b/charts/etcdctl-utility/Chart.yaml
index f28bdaf2..8f12c8ae 100644
--- a/charts/etcdctl-utility/Chart.yaml
+++ b/charts/etcdctl-utility/Chart.yaml
@@ -13,4 +13,4 @@
 apiVersion: v1
 description: etcdctl Client
 name: etcdctl-utility
-version: 0.1.5
+version: 0.1.6
diff --git a/charts/etcdctl-utility/templates/bin/utility/_dbutils.tpl b/charts/etcdctl-utility/templates/bin/utility/_dbutils.tpl
index ff6e9ec0..9f391786 100755
--- a/charts/etcdctl-utility/templates/bin/utility/_dbutils.tpl
+++ b/charts/etcdctl-utility/templates/bin/utility/_dbutils.tpl
@@ -85,7 +85,7 @@ function ensure_ondemand_pod_exists() {
     ONDEMAND_POD=$(echo "$POD_LISTING" | awk '{print $1}')
     STATUS=$(echo "$POD_LISTING" | awk '{print $3}')
     if [[ "$STATUS" == "Terminating" ]]; then
-      kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s
+      kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s 2>/dev/null
       unset ONDEMAND_POD
       unset STATUS
     fi
@@ -115,6 +115,8 @@ function ensure_ondemand_pod_exists() {
   # If we reached this point with no ONDEMAND_POD, then we need to create
   #   a new on-demand job.
   if [[ -z "$ONDEMAND_POD" ]]; then
+    echo "Cleaning up any remainings of previous runs..."
+    do_cleanup "$ONDEMAND_JOB" "$NAMESPACE"
     echo "Creating new on-demand job in the $NAMESPACE namespace..."
     /tmp/etcd-ondemand-job.sh "$NAMESPACE"
     if [[ $? -ne 0 ]]; then
@@ -143,12 +145,12 @@ function ensure_ondemand_pod_exists() {
     ONDEMAND_POD=$(echo "$POD_LISTING" | awk '{print $1}')
     STATUS=$(echo "$POD_LISTING" | awk '{print $3}')
     if [[ "$STATUS" == "Terminating" ]]; then
-      kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s
+      kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s 2>/dev/null
       unset ONDEMAND_POD
     elif [[ "$STATUS" != "Running" ]]; then
-      kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s
+      kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s 2>/dev/null
     elif [[ "$STATUS" != "Pending" ]]; then
-      kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s
+      kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s 2>/dev/null
     fi
   fi
 
@@ -160,7 +162,7 @@ function ensure_ondemand_pod_exists() {
       exit 1
     fi
 
-    kubectl wait --for condition=ready --timeout=300s -n "$NAMESPACE" "pod/${ONDEMAND_POD}"
+    kubectl wait --for condition=ready --timeout=300s -n "$NAMESPACE" "pod/${ONDEMAND_POD}" 2>/dev/null
     if [[ $? -ne 0 ]]; then
       echo "ERROR: Failed to create a new on-demand pod. Exiting..."
       exit 1
@@ -176,20 +178,25 @@ function remove_job() {
   JOB=$1
 
   # Cleanup the last attempted job if there is one, wait for the pod to be deleted.
-  kubectl get job -n "$NAMESPACE" "$JOB"
+  kubectl get job -n "$NAMESPACE" "$JOB" 2>/dev/null
   if [[ $? -eq 0 ]]; then
     echo "Removing on-demand job $NAMESPACE $JOB"
-    ONDEMAND_POD=$(kubectl get pod -n "$NAMESPACE" | grep "$JOB" | awk '{print $1}')
-    kubectl delete job --ignore-not-found -n "$NAMESPACE" "$JOB"
-    kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" job/"${JOB}"
-    if [[ $? -ne 0 ]]; then
-      echo "ERROR: could not destroy the $NAMESPACE $JOB job. Exiting..."
-      exit 1
+    ONDEMAND_JOB=$(kubectl get job -n "$NAMESPACE" | grep "$JOB" | awk '{print $1}')
+    if [[ ! -z ${ONDEMAND_JOB} ]]; then
+      kubectl delete job --ignore-not-found -n "$NAMESPACE" "$JOB"
+      kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" job/"${JOB}" 2>/dev/null
+      if [[ $? -ne 0 ]]; then
+        echo "ERROR: could not destroy the $NAMESPACE $JOB job. Exiting..."
+        exit 1
+      fi
     fi
-    kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" pod/"${ONDEMAND_POD}"
-    if [[ $? -ne 0 ]]; then
-      echo "ERROR: could not destroy the $NAMESPACE $ONDEMAND_POD pod. Exiting..."
-      exit 1
+    ONDEMAND_POD=$(kubectl get pod -n "$NAMESPACE" | grep "$JOB" | awk '{print $1}')
+    if [[ ! -z ${ONDEMAND_POD} ]]; then
+      kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" pod/"${ONDEMAND_POD}"  2>/dev/null
+      if [[ $? -ne 0 ]]; then
+        echo "ERROR: could not destroy the $NAMESPACE $ONDEMAND_POD pod. Exiting..."
+        exit 1
+      fi
     fi
   fi
 }
diff --git a/charts/mysqlclient-utility/Chart.yaml b/charts/mysqlclient-utility/Chart.yaml
index 527463a3..f863e19a 100644
--- a/charts/mysqlclient-utility/Chart.yaml
+++ b/charts/mysqlclient-utility/Chart.yaml
@@ -13,4 +13,4 @@
 apiVersion: v1
 description: Porthole MySql Client
 name: mysqlclient-utility
-version: 0.1.8
+version: 0.1.9
diff --git a/charts/mysqlclient-utility/templates/bin/utility/_dbutils.tpl b/charts/mysqlclient-utility/templates/bin/utility/_dbutils.tpl
index c5d3959c..627eae2a 100755
--- a/charts/mysqlclient-utility/templates/bin/utility/_dbutils.tpl
+++ b/charts/mysqlclient-utility/templates/bin/utility/_dbutils.tpl
@@ -99,7 +99,7 @@ function ensure_ondemand_pod_exists() {
     ONDEMAND_POD=$(echo "$POD_LISTING" | awk '{print $1}')
     STATUS=$(echo "$POD_LISTING" | awk '{print $3}')
     if [[ "$STATUS" == "Terminating" ]]; then
-      kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s
+      kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s 2>/dev/null
       unset ONDEMAND_POD
       unset STATUS
     fi
@@ -129,6 +129,8 @@ function ensure_ondemand_pod_exists() {
   # If we reached this point with no ONDEMAND_POD, then we need to create
   #   a new on-demand job.
   if [[ -z "$ONDEMAND_POD" ]]; then
+    echo "Cleaning up any remainings of previous runs..."
+    do_cleanup "$ONDEMAND_JOB" "$NAMESPACE"
     echo "Creating new on-demand job in the $NAMESPACE namespace..."
     /tmp/mariadb-ondemand-job.sh "$NAMESPACE"
     if [[ $? -ne 0 ]]; then
@@ -149,32 +151,31 @@ function ensure_ondemand_pod_exists() {
       fi
     done
 
-  # Determine the status of the on demand pod if it exists
-  POD_LISTING=$(kubectl get pod --selector application="$ONDEMAND_JOB",component=ondemand --no-headers --namespace "$NAMESPACE")
+  # Determine the status of the on demand pod if it exists POD_LISTING=$(kubectl get pod --selector application="$ONDEMAND_JOB",component=ondemand --no-headers --namespace "$NAMESPACE")
   unset STATUS
   unset ONDEMAND_POD
   if [[ ! -z "$POD_LISTING" ]]; then
     ONDEMAND_POD=$(echo "$POD_LISTING" | awk '{print $1}')
     STATUS=$(echo "$POD_LISTING" | awk '{print $3}')
     if [[ "$STATUS" == "Terminating" ]]; then
-      kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s
+      kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s 2>/dev/null
       unset ONDEMAND_POD
     elif [[ "$STATUS" != "Running" ]]; then
-      kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s
+      kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s 2>/dev/null
     elif [[ "$STATUS" != "Pending" ]]; then
-      kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s
+      kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s 2>/dev/null
     fi
   fi
 
 
 
-    ONDEMAND_POD=$(kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" -o json | jq -r .items[].metadata.name)
+    ONDEMAND_POD=$(kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" -o json  2>/dev/null| jq -r .items[].metadata.name)
     if [[ -z "$ONDEMAND_POD" ]]; then
       echo "ERROR: Failed to obtain the ONDEMAND_POD name."
       exit 1
     fi
 
-    kubectl wait --for condition=ready --timeout=300s -n "$NAMESPACE" "pod/${ONDEMAND_POD}"
+    kubectl wait --for condition=ready --timeout=300s -n "$NAMESPACE" "pod/${ONDEMAND_POD}"  2>/dev/null
     if [[ $? -ne 0 ]]; then
       echo "ERROR: Failed to create a new on-demand pod. Exiting..."
       exit 1
@@ -235,20 +236,25 @@ function remove_job() {
   JOB=$2
 
   # Cleanup the last attempted job if there is one, wait for the pod to be deleted.
-  kubectl get job -n "$NAMESPACE" "$JOB"
+  kubectl get job -n "$NAMESPACE" "$JOB" 2>/dev/null
   if [[ $? -eq 0 ]]; then
     echo "Removing on-demand job $NAMESPACE $JOB"
-    ONDEMAND_POD=$(kubectl get pod -n "$NAMESPACE" | grep "$JOB" | awk '{print $1}')
-    kubectl delete job --ignore-not-found -n "$NAMESPACE" "$JOB"
-    kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" job/"${JOB}"
-    if [[ $? -ne 0 ]]; then
-      echo "ERROR: could not destroy the $NAMESPACE $JOB job. Exiting..."
-      exit 1
-    fi
-    kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" pod/"${ONDEMAND_POD}"
-    if [[ $? -ne 0 ]]; then
-      echo "ERROR: could not destroy the $NAMESPACE $ONDEMAND_POD pod. Exiting..."
-      exit 1
+    ONDEMAND_JOB=$(kubectl get job -n "$NAMESPACE" | grep "$JOB" | awk '{print $1}')
+    if [[ ! -z ${ONDEMAND_JOB} ]]; then
+      kubectl delete job --ignore-not-found -n "$NAMESPACE" "$JOB"
+      kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" job/"${JOB}" 2>/dev/null
+      if [[ $? -ne 0 ]]; then
+        echo "ERROR: could not destroy the $NAMESPACE $JOB job. Exiting..."
+        exit 1
+      fi
+      ONDEMAND_POD=$(kubectl get pod -n "$NAMESPACE" | grep "$JOB" | awk '{print $1}')
+      if [[ ! -z ${ONDEMAND_POD} ]]; then
+        kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" pod/"${ONDEMAND_POD}"  2>/dev/null
+        if [[ $? -ne 0 ]]; then
+          echo "ERROR: could not destroy the $NAMESPACE $ONDEMAND_POD pod. Exiting..."
+          exit 1
+        fi
+      fi
     fi
   fi
 }
@@ -897,6 +903,7 @@ function do_sql_prompt() {
 # Params: [namespace]
 function do_cleanup() {
 
+
   # If a namespace is given go ahead and try to clean it up.
   if [[ ! -z "$2" ]]; then
     remove_job "$2" "$ONDEMAND_JOB"
diff --git a/charts/postgresql-utility/Chart.yaml b/charts/postgresql-utility/Chart.yaml
index e0bca318..ecd762a7 100644
--- a/charts/postgresql-utility/Chart.yaml
+++ b/charts/postgresql-utility/Chart.yaml
@@ -14,4 +14,4 @@
 apiVersion: v1
 description: PostgreSQL Client
 name: postgresql-utility
-version: 0.1.4
+version: 0.1.5
diff --git a/charts/postgresql-utility/templates/bin/utility/_dbutils.tpl b/charts/postgresql-utility/templates/bin/utility/_dbutils.tpl
index 46be3e60..374f370f 100755
--- a/charts/postgresql-utility/templates/bin/utility/_dbutils.tpl
+++ b/charts/postgresql-utility/templates/bin/utility/_dbutils.tpl
@@ -99,7 +99,7 @@ function ensure_ondemand_pod_exists() {
     ONDEMAND_POD=$(echo "$POD_LISTING" | awk '{print $1}')
     STATUS=$(echo "$POD_LISTING" | awk '{print $3}')
     if [[ "$STATUS" == "Terminating" ]]; then
-      kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s
+      kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s 2>/dev/null
       unset ONDEMAND_POD
       unset STATUS
     fi
@@ -129,6 +129,8 @@ function ensure_ondemand_pod_exists() {
   # If we reached this point with no ONDEMAND_POD, then we need to create
   #   a new on-demand job.
   if [[ -z "$ONDEMAND_POD" ]]; then
+    echo "Cleaning up any remainings of previous runs..."
+    do_cleanup "$ONDEMAND_JOB" "$NAMESPACE"
     echo "Creating new on-demand job in the $NAMESPACE namespace..."
     /tmp/pg-ondemand-job.sh "$NAMESPACE"
     if [[ $? -ne 0 ]]; then
@@ -157,24 +159,24 @@ function ensure_ondemand_pod_exists() {
     ONDEMAND_POD=$(echo "$POD_LISTING" | awk '{print $1}')
     STATUS=$(echo "$POD_LISTING" | awk '{print $3}')
     if [[ "$STATUS" == "Terminating" ]]; then
-      kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s
+      kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s 2>/dev/null
       unset ONDEMAND_POD
     elif [[ "$STATUS" != "Running" ]]; then
-      kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s
+      kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s 2>/dev/null
     elif [[ "$STATUS" != "Pending" ]]; then
-      kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s
+      kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s 2>/dev/null
     fi
   fi
 
 
 
-    ONDEMAND_POD=$(kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" -o json | jq -r .items[].metadata.name)
+    ONDEMAND_POD=$(kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" -o json  2>/dev/null| jq -r .items[].metadata.name)
     if [[ -z "$ONDEMAND_POD" ]]; then
       echo "ERROR: Failed to obtain the ONDEMAND_POD name."
       exit 1
     fi
 
-    kubectl wait --for condition=ready --timeout=300s -n "$NAMESPACE" "pod/${ONDEMAND_POD}"
+    kubectl wait --for condition=ready --timeout=300s -n "$NAMESPACE" "pod/${ONDEMAND_POD}"  2>/dev/null
     if [[ $? -ne 0 ]]; then
       echo "ERROR: Failed to create a new on-demand pod. Exiting..."
       exit 1
@@ -234,20 +236,25 @@ function remove_job() {
   JOB=$2
 
   # Cleanup the last attempted job if there is one, wait for the pod to be deleted.
-  kubectl get job -n "$NAMESPACE" "$JOB"
+  kubectl get job -n "$NAMESPACE" "$JOB" 2>/dev/null
   if [[ $? -eq 0 ]]; then
     echo "Removing on-demand job $NAMESPACE $JOB"
-    ONDEMAND_POD=$(kubectl get pod -n "$NAMESPACE" | grep "$JOB" | awk '{print $1}')
-    kubectl delete job --ignore-not-found -n "$NAMESPACE" "$JOB"
-    kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" job/"${JOB}"
-    if [[ $? -ne 0 ]]; then
-      echo "ERROR: could not destroy the $NAMESPACE $JOB job. Exiting..."
-      exit 1
+    ONDEMAND_JOB=$(kubectl get job -n "$NAMESPACE" | grep "$JOB" | awk '{print $1}')
+    if [[ ! -z ${ONDEMAND_JOB} ]]; then
+      kubectl delete job --ignore-not-found -n "$NAMESPACE" "$JOB"
+      kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" job/"${JOB}" 2>/dev/null
+      if [[ $? -ne 0 ]]; then
+        echo "ERROR: could not destroy the $NAMESPACE $JOB job. Exiting..."
+        exit 1
+      fi
     fi
-    kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" pod/"${ONDEMAND_POD}"
-    if [[ $? -ne 0 ]]; then
-      echo "ERROR: could not destroy the $NAMESPACE $ONDEMAND_POD pod. Exiting..."
-      exit 1
+    ONDEMAND_POD=$(kubectl get pod -n "$NAMESPACE" | grep "$JOB" | awk '{print $1}')
+    if [[ ! -z ${ONDEMAND_POD} ]]; then
+      kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" pod/"${ONDEMAND_POD}"  2>/dev/null
+      if [[ $? -ne 0 ]]; then
+        echo "ERROR: could not destroy the $NAMESPACE $ONDEMAND_POD pod. Exiting..."
+        exit 1
+      fi
     fi
   fi
 }