[dbutils] Wait for ON_DEMAND_POD to be created

This PS is intended to fix intermittent issue on some sites when
database backup ondemand job fails to detect ON_DEMAND_POD when it is
still creating.

Change-Id: I4940a5f5a4d0739d8ccc9e55d042dfc22884e789

charts/etcdctl-utility/Chart.yaml

@@ -13,4 +13,4 @@
 apiVersion: v1
 description: etcdctl Client
 name: etcdctl-utility
-version: 0.1.2
+version: 0.1.3

charts/etcdctl-utility/templates/bin/utility/_dbutils.tpl

@@ -119,7 +119,19 @@ function ensure_ondemand_pod_exists() {
       exit 1
     fi
 
-    sleep 10s
+    # waiting for ondemand pod to be created
+    RETRIES=10
+    until kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB"  | grep ondemand; do
+      RETRIES=$((RETRIES-1))
+      if [ ${RETRIES} -ge 1 ]; then
+        echo "ONDEMAND_POD is being created... Waiting for 10 seconds... Retries left ${RETRIES}..."
+        sleep 10s
+      else
+        echo "ERROR: Failed to create a new on-demand pod. Exiting..."
+        exit 1
+      fi
+    done
+
     ONDEMAND_POD=$(kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" -o json | jq -r .items[].metadata.name)
     if [[ -z "$ONDEMAND_POD" ]]; then
       echo "ERROR: Failed to obtain the ONDEMAND_POD name."

charts/mysqlclient-utility/Chart.yaml

@@ -13,4 +13,4 @@
 apiVersion: v1
 description: Porthole MySql Client
 name: mysqlclient-utility
-version: 0.1.2
+version: 0.1.3

charts/mysqlclient-utility/templates/bin/utility/_dbutils.tpl

@@ -133,7 +133,19 @@ function ensure_ondemand_pod_exists() {
       exit 1
     fi
 
-    sleep 10s
+    # waiting for ondemand pod to be created
+    RETRIES=10
+    until kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB"  | grep ondemand; do
+      RETRIES=$((RETRIES-1))
+      if [ ${RETRIES} -ge 1 ]; then
+        echo "ONDEMAND_POD is being created... Waiting for 10 seconds... Retries left ${RETRIES}..."
+        sleep 10s
+      else
+        echo "ERROR: Failed to create a new on-demand pod. Exiting..."
+        exit 1
+      fi
+    done
+
     ONDEMAND_POD=$(kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" -o json | jq -r .items[].metadata.name)
     if [[ -z "$ONDEMAND_POD" ]]; then
       echo "ERROR: Failed to obtain the ONDEMAND_POD name."

charts/postgresql-utility/Chart.yaml

@@ -14,4 +14,4 @@
 apiVersion: v1
 description: PostgreSQL Client
 name: postgresql-utility
-version: 0.1.1
+version: 0.1.2

charts/postgresql-utility/templates/bin/utility/_dbutils.tpl

@@ -133,7 +133,19 @@ function ensure_ondemand_pod_exists() {
       exit 1
     fi
 
-    sleep 10s
+    # waiting for ondemand pod to be created
+    RETRIES=10
+    until kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB"  | grep ondemand; do
+      RETRIES=$((RETRIES-1))
+      if [ ${RETRIES} -ge 1 ]; then
+        echo "ONDEMAND_POD is being created... Waiting for 10 seconds... Retries left ${RETRIES}..."
+        sleep 10s
+      else
+        echo "ERROR: Failed to create a new on-demand pod. Exiting..."
+        exit 1
+      fi
+    done
+
     ONDEMAND_POD=$(kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" -o json | jq -r .items[].metadata.name)
     if [[ -z "$ONDEMAND_POD" ]]; then
       echo "ERROR: Failed to obtain the ONDEMAND_POD name."

images/mysqlclient-utility/Dockerfile.ubuntu_focal

@@ -46,7 +46,7 @@ RUN set -xe \
     && apt-get clean -y \
     && curl --silent -L https://dl.k8s.io/v${KUBE_VERSION}/kubernetes-client-linux-amd64.tar.gz \
        | tar -zC /usr/bin --strip-components=3 --wildcards -x "*/*/*/kubectl"  \
-    && curl --silent -L https://percona.com/get/percona-toolkit.tar.gz \
+    && curl --silent -L https://downloads.percona.com/downloads/percona-toolkit/3.5.1/binary/tarball/percona-toolkit-3.5.1_x86_64.tar.gz \
        | tar -zC /usr/local/bin/ --strip-components=2 --wildcards -x "*/*/pt-show-grants" \
     && rm -rf \
       /tmp/* \