From f0efd75c99f2fe86c332f9b2662b1bf17b6f086c Mon Sep 17 00:00:00 2001
From: Sergiy Markin <smarkin@mirantis.com>
Date: Tue, 5 Dec 2023 21:41:19 +0000
Subject: [PATCH] [utility] Ondemand jobs improvements

This PS is intended to address several issues related to ondemand pod
management having the goal to improve testing process:

- replaces 1 million seconds delay in main container with
  configurable value set by default to 3,600 seconds
- improvement in dbutils.sh script that cerated ondemand job and waits
  for ondemand pod creation

The changes are affecting etcdctl-utility, mysqlclient-utility and
postgresql-utility charts.

Change-Id: I9a2f2c6d2e84414cbb3e175c0b94f9780e8a6388
---
 charts/etcdctl-utility/Chart.yaml             |  2 +-
 .../templates/bin/utility/_dbutils.tpl        | 65 ++++++++++++-----
 .../bin/utility/_etcd_ondemand_job.sh.tpl     |  2 +-
 charts/etcdctl-utility/values.yaml            |  2 +
 charts/mysqlclient-utility/Chart.yaml         |  2 +-
 .../templates/bin/utility/_dbutils.tpl        | 69 +++++++++++++------
 .../bin/utility/_mariadb_ondemand_job.sh.tpl  | 13 ++--
 charts/mysqlclient-utility/values.yaml        |  2 +
 charts/postgresql-utility/Chart.yaml          |  2 +-
 .../templates/bin/utility/_dbutils.tpl        | 65 ++++++++++++-----
 .../bin/utility/_pg_ondemand_job.sh.tpl       |  2 +-
 charts/postgresql-utility/values.yaml         |  2 +
 12 files changed, 158 insertions(+), 70 deletions(-)

diff --git a/charts/etcdctl-utility/Chart.yaml b/charts/etcdctl-utility/Chart.yaml
index 729d2090..8cdf8cbb 100644
--- a/charts/etcdctl-utility/Chart.yaml
+++ b/charts/etcdctl-utility/Chart.yaml
@@ -13,4 +13,4 @@
 apiVersion: v1
 description: etcdctl Client
 name: etcdctl-utility
-version: 0.1.3
+version: 0.1.4
diff --git a/charts/etcdctl-utility/templates/bin/utility/_dbutils.tpl b/charts/etcdctl-utility/templates/bin/utility/_dbutils.tpl
index 018865fb..ff6e9ec0 100755
--- a/charts/etcdctl-utility/templates/bin/utility/_dbutils.tpl
+++ b/charts/etcdctl-utility/templates/bin/utility/_dbutils.tpl
@@ -78,19 +78,20 @@ function check_args() {
 function ensure_ondemand_pod_exists() {
 
   # Determine the status of the on demand pod if it exists
-  POD_LISTING=$(kubectl get pod -n "$NAMESPACE" | grep "$ONDEMAND_JOB")
+  POD_LISTING=$(kubectl get pod --selector application="$ONDEMAND_JOB",component=ondemand --no-headers --namespace "$NAMESPACE" 2>/dev/null)
+  unset STATUS
+  unset ONDEMAND_POD
   if [[ ! -z "$POD_LISTING" ]]; then
     ONDEMAND_POD=$(echo "$POD_LISTING" | awk '{print $1}')
     STATUS=$(echo "$POD_LISTING" | awk '{print $3}')
     if [[ "$STATUS" == "Terminating" ]]; then
-      kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=30s
+      kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s
       unset ONDEMAND_POD
-    elif [[ "$STATUS" != "Running" ]]; then
-      kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=30s
+      unset STATUS
     fi
   fi
 
-  POD_LISTING=$(kubectl get pod -n "$NAMESPACE" | grep "$ONDEMAND_JOB")
+  POD_LISTING=$(kubectl get pod --selector application="$ONDEMAND_JOB",component=ondemand --no-headers --namespace "$NAMESPACE" 2>/dev/null)
   if [[ ! -z "$POD_LISTING" ]]; then
     STATUS=$(echo "$POD_LISTING" | awk '{print $3}')
     CONTAINERS=$(echo "$POD_LISTING" | awk '{print $2}')
@@ -98,14 +99,16 @@ function ensure_ondemand_pod_exists() {
     #    we find any which are not ready remove them, even if completed.
     if [[ $STATUS != "Running" || $CONTAINERS != "1/1" ]]; then
       echo "Found an old on-demand pod; removing it."
-      remove_job "$NAMESPACE" "$ONDEMAND_JOB"
+      remove_job "$ONDEMAND_JOB"
       if [[ $? -ne 0 ]]; then
         echo "ERROR: Failed to remove old on-demand pod. Exiting..."
         exit 1
       fi
+      unset ONDEMAND_POD
+      unset STATUS
     else
       # Pod is already running and ready
-      ONDEMAND_POD=$(kubectl get pod -n "$NAMESPACE" | grep "$ONDEMAND_JOB" | awk '{print $1}')
+      ONDEMAND_POD=$(kubectl get pod --selector application="$ONDEMAND_JOB",component=ondemand --no-headers --namespace "$NAMESPACE" | awk '{print $1}')
     fi
   fi
 
@@ -119,19 +122,38 @@ function ensure_ondemand_pod_exists() {
       exit 1
     fi
 
-    # waiting for ondemand pod to be created
+    # waiting for ondemand job to be created
     RETRIES=10
-    until kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB"  | grep ondemand; do
+    until kubectl get job -n "$NAMESPACE" --no-headers  | grep "$ONDEMAND_JOB"; do
       RETRIES=$((RETRIES-1))
       if [ ${RETRIES} -ge 1 ]; then
-        echo "ONDEMAND_POD is being created... Waiting for 10 seconds... Retries left ${RETRIES}..."
+        echo "ONDEMAND_JOB is being created... Waiting for 10 seconds... Retries left ${RETRIES}..."
         sleep 10s
       else
-        echo "ERROR: Failed to create a new on-demand pod. Exiting..."
+        echo "ERROR: Failed to create a new on-demand job. Exiting..."
         exit 1
       fi
     done
 
+  # Determine the status of the on demand pod if it exists
+  POD_LISTING=$(kubectl get pod --selector application="$ONDEMAND_JOB",component=ondemand --no-headers --namespace "$NAMESPACE")
+  unset STATUS
+  unset ONDEMAND_POD
+  if [[ ! -z "$POD_LISTING" ]]; then
+    ONDEMAND_POD=$(echo "$POD_LISTING" | awk '{print $1}')
+    STATUS=$(echo "$POD_LISTING" | awk '{print $3}')
+    if [[ "$STATUS" == "Terminating" ]]; then
+      kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s
+      unset ONDEMAND_POD
+    elif [[ "$STATUS" != "Running" ]]; then
+      kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s
+    elif [[ "$STATUS" != "Pending" ]]; then
+      kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s
+    fi
+  fi
+
+
+
     ONDEMAND_POD=$(kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" -o json | jq -r .items[].metadata.name)
     if [[ -z "$ONDEMAND_POD" ]]; then
       echo "ERROR: Failed to obtain the ONDEMAND_POD name."
@@ -148,7 +170,7 @@ function ensure_ondemand_pod_exists() {
   export ONDEMAND_POD
 }
 
-# Params: <job>
+# Params: [namespace] <job>
 function remove_job() {
 
   JOB=$1
@@ -157,13 +179,18 @@ function remove_job() {
   kubectl get job -n "$NAMESPACE" "$JOB"
   if [[ $? -eq 0 ]]; then
     echo "Removing on-demand job $NAMESPACE $JOB"
-    ONDEMAND_POD=$(kubectl get pod -n "$NAMESPACE" | grep "$ONDEMAND_JOB" | awk '{print $1}')
+    ONDEMAND_POD=$(kubectl get pod -n "$NAMESPACE" | grep "$JOB" | awk '{print $1}')
     kubectl delete job --ignore-not-found -n "$NAMESPACE" "$JOB"
-    kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" pod/"${ONDEMAND_POD}" &>/dev/null
+    kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" job/"${JOB}"
     if [[ $? -ne 0 ]]; then
       echo "ERROR: could not destroy the $NAMESPACE $JOB job. Exiting..."
       exit 1
     fi
+    kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" pod/"${ONDEMAND_POD}"
+    if [[ $? -ne 0 ]]; then
+      echo "ERROR: could not destroy the $NAMESPACE $ONDEMAND_POD pod. Exiting..."
+      exit 1
+    fi
   fi
 }
 
@@ -181,7 +208,7 @@ function do_backup() {
   ensure_ondemand_pod_exists
 
   # Execute the command in the on-demand pod
-  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- /tmp/backup_etcd.sh
+  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- /tmp/backup_etcd.sh
 }
 
 # Params: [-rp] <node>
@@ -198,7 +225,7 @@ function do_list_archives() {
   ensure_ondemand_pod_exists
 
   # Execute the command in the on-demand pod
-  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- /tmp/restore_etcd.sh list_archives "$LOCATION"
+  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- /tmp/restore_etcd.sh list_archives "$LOCATION"
 }
 
 # Params: [-rp] <archive> <anchor> <node>
@@ -231,7 +258,7 @@ function do_restore() {
   ensure_ondemand_pod_exists
 
   # Execute the command in the on-demand pod
-  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- /tmp/restore_etcd.sh restore "$ARCHIVE" "$DATABASE" "$LOCATION"
+  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- /tmp/restore_etcd.sh restore "$ARCHIVE" "$DATABASE" "$LOCATION"
 }
 
 # Params: [-rp] <archive> <node>
@@ -262,13 +289,13 @@ function do_delete_archive() {
   ensure_ondemand_pod_exists
 
   # Execute the command in the on-demand pod
-  kubectl exec -i -n "${NAMESPACE}" "${ONDEMAND_POD}" -- /tmp/restore_etcd.sh delete_archive "${ARCHIVE}" "${LOCATION}"
+  kubectl exec -i -n "${NAMESPACE}" "${ONDEMAND_POD}" -c "${ONDEMAND_JOB}" -- /tmp/restore_etcd.sh delete_archive "${ARCHIVE}" "${LOCATION}"
 }
 
 function do_cleanup() {
 
   if [[ "$KEEP_POD" == "false" ]]; then
-    remove_job "$ONDEMAND_JOB"
+    remove_job  "${ONDEMAND_JOB}"
 
     unset ONDEMAND_POD
 
diff --git a/charts/etcdctl-utility/templates/bin/utility/_etcd_ondemand_job.sh.tpl b/charts/etcdctl-utility/templates/bin/utility/_etcd_ondemand_job.sh.tpl
index f4b45112..58e45c95 100644
--- a/charts/etcdctl-utility/templates/bin/utility/_etcd_ondemand_job.sh.tpl
+++ b/charts/etcdctl-utility/templates/bin/utility/_etcd_ondemand_job.sh.tpl
@@ -56,7 +56,7 @@ spec:
 {{ dict "envAll" $envAll "application" "etcd_ondemand" "container" "etcd_ondemand" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
           command:
             - /bin/sleep
-            - "1000000"
+            - "{{ .Values.conf.etcd_ondemand.ondemapd_pod_sleep_time }}"
           env:
             - name: ETCDCTL_API
               value: "{{ .Values.conf.etcd.etcdctl_api }}"
diff --git a/charts/etcdctl-utility/values.yaml b/charts/etcdctl-utility/values.yaml
index 98a4e9fd..5f96844f 100644
--- a/charts/etcdctl-utility/values.yaml
+++ b/charts/etcdctl-utility/values.yaml
@@ -142,6 +142,8 @@ conf:
       dbutils: CommandFilter, dbutils, nobody
       etcdctl: CommandFilter, etcdctl, root
       kubectl: CommandFilter, kubectl, root
+  etcd_ondemand:
+    ondemapd_pod_sleep_time: 3600
   etcdctlrootwrapconf:
     DEFAULT:
       # Configuration for etcdctl-rootwrap
diff --git a/charts/mysqlclient-utility/Chart.yaml b/charts/mysqlclient-utility/Chart.yaml
index 4de8d3e2..3e6aff91 100644
--- a/charts/mysqlclient-utility/Chart.yaml
+++ b/charts/mysqlclient-utility/Chart.yaml
@@ -13,4 +13,4 @@
 apiVersion: v1
 description: Porthole MySql Client
 name: mysqlclient-utility
-version: 0.1.4
+version: 0.1.5
diff --git a/charts/mysqlclient-utility/templates/bin/utility/_dbutils.tpl b/charts/mysqlclient-utility/templates/bin/utility/_dbutils.tpl
index befbbbac..7ebb39fd 100755
--- a/charts/mysqlclient-utility/templates/bin/utility/_dbutils.tpl
+++ b/charts/mysqlclient-utility/templates/bin/utility/_dbutils.tpl
@@ -92,19 +92,20 @@ function check_args() {
 function ensure_ondemand_pod_exists() {
 
   # Determine the status of the on demand pod if it exists
-  POD_LISTING=$(kubectl get pod -n "$NAMESPACE" | grep "$ONDEMAND_JOB")
+  POD_LISTING=$(kubectl get pod --selector application="$ONDEMAND_JOB",component=ondemand --no-headers --namespace "$NAMESPACE" 2>/dev/null)
+  unset STATUS
+  unset ONDEMAND_POD
   if [[ ! -z "$POD_LISTING" ]]; then
     ONDEMAND_POD=$(echo "$POD_LISTING" | awk '{print $1}')
     STATUS=$(echo "$POD_LISTING" | awk '{print $3}')
     if [[ "$STATUS" == "Terminating" ]]; then
-      kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=30s
+      kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s
       unset ONDEMAND_POD
-    elif [[ "$STATUS" != "Running" ]]; then
-      kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=30s
+      unset STATUS
     fi
   fi
 
-  POD_LISTING=$(kubectl get pod -n "$NAMESPACE" | grep "$ONDEMAND_JOB")
+  POD_LISTING=$(kubectl get pod --selector application="$ONDEMAND_JOB",component=ondemand --no-headers --namespace "$NAMESPACE" 2>/dev/null)
   if [[ ! -z "$POD_LISTING" ]]; then
     STATUS=$(echo "$POD_LISTING" | awk '{print $3}')
     CONTAINERS=$(echo "$POD_LISTING" | awk '{print $2}')
@@ -117,9 +118,11 @@ function ensure_ondemand_pod_exists() {
         echo "ERROR: Failed to remove old on-demand pod. Exiting..."
         exit 1
       fi
+      unset ONDEMAND_POD
+      unset STATUS
     else
       # Pod is already running and ready
-      ONDEMAND_POD=$(kubectl get pod -n "$NAMESPACE" | grep "$ONDEMAND_JOB" | awk '{print $1}')
+      ONDEMAND_POD=$(kubectl get pod --selector application="$ONDEMAND_JOB",component=ondemand --no-headers --namespace "$NAMESPACE" | awk '{print $1}')
     fi
   fi
 
@@ -133,19 +136,38 @@ function ensure_ondemand_pod_exists() {
       exit 1
     fi
 
-    # waiting for ondemand pod to be created
-    RETRIES=10
-    until kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB"  | grep ondemand; do
+    # waiting for ondemand job to be created
+    RETRIES=30
+    until kubectl get job -n "$NAMESPACE" --no-headers  | grep "$ONDEMAND_JOB"; do
       RETRIES=$((RETRIES-1))
       if [ ${RETRIES} -ge 1 ]; then
-        echo "ONDEMAND_POD is being created... Waiting for 10 seconds... Retries left ${RETRIES}..."
+        echo "ONDEMAND_JOB is being created... Waiting for 10 seconds... Retries left ${RETRIES}..."
         sleep 10s
       else
-        echo "ERROR: Failed to create a new on-demand pod. Exiting..."
+        echo "ERROR: Failed to create a new on-demand job. Exiting..."
         exit 1
       fi
     done
 
+  # Determine the status of the on demand pod if it exists
+  POD_LISTING=$(kubectl get pod --selector application="$ONDEMAND_JOB",component=ondemand --no-headers --namespace "$NAMESPACE")
+  unset STATUS
+  unset ONDEMAND_POD
+  if [[ ! -z "$POD_LISTING" ]]; then
+    ONDEMAND_POD=$(echo "$POD_LISTING" | awk '{print $1}')
+    STATUS=$(echo "$POD_LISTING" | awk '{print $3}')
+    if [[ "$STATUS" == "Terminating" ]]; then
+      kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s
+      unset ONDEMAND_POD
+    elif [[ "$STATUS" != "Running" ]]; then
+      kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s
+    elif [[ "$STATUS" != "Pending" ]]; then
+      kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s
+    fi
+  fi
+
+
+
     ONDEMAND_POD=$(kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" -o json | jq -r .items[].metadata.name)
     if [[ -z "$ONDEMAND_POD" ]]; then
       echo "ERROR: Failed to obtain the ONDEMAND_POD name."
@@ -206,7 +228,7 @@ function setup_namespace() {
   return 0
 }
 
-# Params: <namespace> <job>
+# Params: [namespace] <job>
 function remove_job() {
 
   NAMESPACE=$1
@@ -218,11 +240,16 @@ function remove_job() {
     echo "Removing on-demand job $NAMESPACE $JOB"
     ONDEMAND_POD=$(kubectl get pod -n "$NAMESPACE" | grep "$JOB" | awk '{print $1}')
     kubectl delete job --ignore-not-found -n "$NAMESPACE" "$JOB"
-    kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" pod/"${ONDEMAND_POD}" &>/dev/null
+    kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" job/"${JOB}"
     if [[ $? -ne 0 ]]; then
       echo "ERROR: could not destroy the $NAMESPACE $JOB job. Exiting..."
       exit 1
     fi
+    kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" pod/"${ONDEMAND_POD}"
+    if [[ $? -ne 0 ]]; then
+      echo "ERROR: could not destroy the $NAMESPACE $ONDEMAND_POD pod. Exiting..."
+      exit 1
+    fi
   fi
 }
 
@@ -285,7 +312,7 @@ function do_backup() {
     COMMAND="/tmp/backup_mariadb.sh ${BACKUP_ARGS[${DB_LOC}]}"
   fi
 
-  kubectl exec -i -n "${NAMESPACE}" "${ONDEMAND_POD}" -- ${COMMAND}
+  kubectl exec -i -n "${NAMESPACE}" "${ONDEMAND_POD}" -c "${ONDEMAND_JOB}" -- ${COMMAND}
 
   unlock
 }
@@ -304,7 +331,7 @@ function do_list_archives() {
   ensure_ondemand_pod_exists
 
   # Execute the command in the on-demand pod
-  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- /tmp/restore_mariadb.sh list_archives "$LOCATION"
+  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- /tmp/restore_mariadb.sh list_archives "$LOCATION"
 }
 
 # Params: [-rp] <archive>
@@ -336,7 +363,7 @@ function do_list_databases() {
   ensure_ondemand_pod_exists
 
   # Execute the command in the on-demand pod
-  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- /tmp/restore_mariadb.sh list_databases "$ARCHIVE" "$LOCATION"
+  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- /tmp/restore_mariadb.sh list_databases "$ARCHIVE" "$LOCATION"
 }
 
 # Params: [-rp] <archive> <database>
@@ -370,7 +397,7 @@ function do_list_tables() {
   ensure_ondemand_pod_exists
 
   # Execute the command in the on-demand pod
-  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- /tmp/restore_mariadb.sh list_tables "$ARCHIVE" "$DATABASE" "$LOCATION"
+  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- /tmp/restore_mariadb.sh list_tables "$ARCHIVE" "$DATABASE" "$LOCATION"
 }
 
 # Params: [-rp] <archive> <database> <table>
@@ -404,7 +431,7 @@ function do_list_rows() {
   ensure_ondemand_pod_exists
 
   # Execute the command in the on-demand pod
-  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- /tmp/restore_mariadb.sh list_rows "$ARCHIVE" "$DATABASE" "$TABLE" "$LOCATION"
+  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- /tmp/restore_mariadb.sh list_rows "$ARCHIVE" "$DATABASE" "$TABLE" "$LOCATION"
 }
 
 # Params: [-rp] <archive> <database> <table>
@@ -438,7 +465,7 @@ function do_list_schema() {
   ensure_ondemand_pod_exists
 
   # Execute the command in the on-demand pod
-  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- /tmp/restore_mariadb.sh list_schema "$ARCHIVE" "$DATABASE" "$TABLE" "$LOCATION"
+  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- /tmp/restore_mariadb.sh list_schema "$ARCHIVE" "$DATABASE" "$TABLE" "$LOCATION"
 }
 
 # Params: [-rp] <archive>
@@ -470,7 +497,7 @@ function do_delete_archive() {
   ensure_ondemand_pod_exists
 
   # Execute the command in the on-demand pod
-  kubectl exec -i -n "${NAMESPACE}" "${ONDEMAND_POD}" -- /tmp/restore_mariadb.sh delete_archive "${ARCHIVE}" "${LOCATION}"
+  kubectl exec -i -n "${NAMESPACE}" "${ONDEMAND_POD}" -c "${ONDEMAND_JOB}" -- /tmp/restore_mariadb.sh delete_archive "${ARCHIVE}" "${LOCATION}"
 }
 
 # Params: [-p] <namespace>
@@ -838,7 +865,7 @@ function do_restore() {
   lock 300
 
   # Execute the command in the on-demand pod
-  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- /tmp/restore_mariadb.sh restore "$ARCHIVE" "$DATABASE" "$LOCATION"
+  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- /tmp/restore_mariadb.sh restore "$ARCHIVE" "$DATABASE" "$LOCATION"
 
   unlock
 }
diff --git a/charts/mysqlclient-utility/templates/bin/utility/_mariadb_ondemand_job.sh.tpl b/charts/mysqlclient-utility/templates/bin/utility/_mariadb_ondemand_job.sh.tpl
index bf2cf9db..5037dc1d 100644
--- a/charts/mysqlclient-utility/templates/bin/utility/_mariadb_ondemand_job.sh.tpl
+++ b/charts/mysqlclient-utility/templates/bin/utility/_mariadb_ondemand_job.sh.tpl
@@ -92,11 +92,9 @@ spec:
           image: ${MYSQLCLIENT_UTILTIY_IMAGE_NAME}
 {{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_ondemand | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
 {{ dict "envAll" $envAll "application" "mariadb_ondemand" "container" "mariadb_ondemand" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
-          command: ["/bin/sh"]
-          args:
-            - -c
-            - >-
-                /bin/sleep "1000000"
+          command:
+            - /bin/sleep
+            - "{{ .Values.conf.mariadb_ondemand.ondemapd_pod_sleep_time }}"
           env:
             - name: MARIADB_BACKUP_BASE_DIR
               valueFrom:
@@ -246,7 +244,10 @@ if $TLS_ENABLED; then
             - name: MYSQL_HISTFILE
               value: /dev/null
           command:
-            - /tmp/start_verification_server.sh
+            - /bin/sh
+          args:
+            - -c
+            - ( /tmp/start_verification_server.sh )& /bin/sleep {{ .Values.conf.mariadb_ondemand.ondemapd_pod_sleep_time }}
           volumeMounts:
             - name: pod-tmp
               mountPath: /tmp
diff --git a/charts/mysqlclient-utility/values.yaml b/charts/mysqlclient-utility/values.yaml
index 8aec8b8a..e7660889 100644
--- a/charts/mysqlclient-utility/values.yaml
+++ b/charts/mysqlclient-utility/values.yaml
@@ -172,6 +172,8 @@ conf:
       mysql: CommandFilter, mysql, root
       kubectl: CommandFilter, kubectl, root
       dbutils: CommandFilter, dbutils, nobody
+  mariadb_ondemand:
+    ondemapd_pod_sleep_time: 3600
   mysqlclientrootwrapconf:
     DEFAULT:
       # Configuration for mysqlclient-rootwrap
diff --git a/charts/postgresql-utility/Chart.yaml b/charts/postgresql-utility/Chart.yaml
index b7012790..03b5cccd 100644
--- a/charts/postgresql-utility/Chart.yaml
+++ b/charts/postgresql-utility/Chart.yaml
@@ -14,4 +14,4 @@
 apiVersion: v1
 description: PostgreSQL Client
 name: postgresql-utility
-version: 0.1.2
+version: 0.1.3
diff --git a/charts/postgresql-utility/templates/bin/utility/_dbutils.tpl b/charts/postgresql-utility/templates/bin/utility/_dbutils.tpl
index 03c7bc0d..46be3e60 100755
--- a/charts/postgresql-utility/templates/bin/utility/_dbutils.tpl
+++ b/charts/postgresql-utility/templates/bin/utility/_dbutils.tpl
@@ -92,19 +92,20 @@ function check_args() {
 function ensure_ondemand_pod_exists() {
 
   # Determine the status of the on demand pod if it exists
-  POD_LISTING=$(kubectl get pod -n "$NAMESPACE" | grep "$ONDEMAND_JOB")
+  POD_LISTING=$(kubectl get pod --selector application="$ONDEMAND_JOB",component=ondemand --no-headers --namespace "$NAMESPACE" 2>/dev/null)
+  unset STATUS
+  unset ONDEMAND_POD
   if [[ ! -z "$POD_LISTING" ]]; then
     ONDEMAND_POD=$(echo "$POD_LISTING" | awk '{print $1}')
     STATUS=$(echo "$POD_LISTING" | awk '{print $3}')
     if [[ "$STATUS" == "Terminating" ]]; then
-      kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=30s
+      kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s
       unset ONDEMAND_POD
-    elif [[ "$STATUS" != "Running" ]]; then
-      kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=30s
+      unset STATUS
     fi
   fi
 
-  POD_LISTING=$(kubectl get pod -n "$NAMESPACE" | grep "$ONDEMAND_JOB")
+  POD_LISTING=$(kubectl get pod --selector application="$ONDEMAND_JOB",component=ondemand --no-headers --namespace "$NAMESPACE" 2>/dev/null)
   if [[ ! -z "$POD_LISTING" ]]; then
     STATUS=$(echo "$POD_LISTING" | awk '{print $3}')
     CONTAINERS=$(echo "$POD_LISTING" | awk '{print $2}')
@@ -117,9 +118,11 @@ function ensure_ondemand_pod_exists() {
         echo "ERROR: Failed to remove old on-demand pod. Exiting..."
         exit 1
       fi
+      unset ONDEMAND_POD
+      unset STATUS
     else
       # Pod is already running and ready
-      ONDEMAND_POD=$(kubectl get pod -n "$NAMESPACE" | grep "$ONDEMAND_JOB" | awk '{print $1}')
+      ONDEMAND_POD=$(kubectl get pod --selector application="$ONDEMAND_JOB",component=ondemand --no-headers --namespace "$NAMESPACE" | awk '{print $1}')
     fi
   fi
 
@@ -133,19 +136,38 @@ function ensure_ondemand_pod_exists() {
       exit 1
     fi
 
-    # waiting for ondemand pod to be created
+    # waiting for ondemand job to be created
     RETRIES=10
-    until kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB"  | grep ondemand; do
+    until kubectl get job -n "$NAMESPACE" --no-headers  | grep "$ONDEMAND_JOB"; do
       RETRIES=$((RETRIES-1))
       if [ ${RETRIES} -ge 1 ]; then
-        echo "ONDEMAND_POD is being created... Waiting for 10 seconds... Retries left ${RETRIES}..."
+        echo "ONDEMAND_JOB is being created... Waiting for 10 seconds... Retries left ${RETRIES}..."
         sleep 10s
       else
-        echo "ERROR: Failed to create a new on-demand pod. Exiting..."
+        echo "ERROR: Failed to create a new on-demand job. Exiting..."
         exit 1
       fi
     done
 
+  # Determine the status of the on demand pod if it exists
+  POD_LISTING=$(kubectl get pod --selector application="$ONDEMAND_JOB",component=ondemand --no-headers --namespace "$NAMESPACE")
+  unset STATUS
+  unset ONDEMAND_POD
+  if [[ ! -z "$POD_LISTING" ]]; then
+    ONDEMAND_POD=$(echo "$POD_LISTING" | awk '{print $1}')
+    STATUS=$(echo "$POD_LISTING" | awk '{print $3}')
+    if [[ "$STATUS" == "Terminating" ]]; then
+      kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s
+      unset ONDEMAND_POD
+    elif [[ "$STATUS" != "Running" ]]; then
+      kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s
+    elif [[ "$STATUS" != "Pending" ]]; then
+      kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s
+    fi
+  fi
+
+
+
     ONDEMAND_POD=$(kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" -o json | jq -r .items[].metadata.name)
     if [[ -z "$ONDEMAND_POD" ]]; then
       echo "ERROR: Failed to obtain the ONDEMAND_POD name."
@@ -217,11 +239,16 @@ function remove_job() {
     echo "Removing on-demand job $NAMESPACE $JOB"
     ONDEMAND_POD=$(kubectl get pod -n "$NAMESPACE" | grep "$JOB" | awk '{print $1}')
     kubectl delete job --ignore-not-found -n "$NAMESPACE" "$JOB"
-    kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" pod/"${ONDEMAND_POD}" &>/dev/null
+    kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" job/"${JOB}"
     if [[ $? -ne 0 ]]; then
       echo "ERROR: could not destroy the $NAMESPACE $JOB job. Exiting..."
       exit 1
     fi
+    kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" pod/"${ONDEMAND_POD}"
+    if [[ $? -ne 0 ]]; then
+      echo "ERROR: could not destroy the $NAMESPACE $ONDEMAND_POD pod. Exiting..."
+      exit 1
+    fi
   fi
 }
 
@@ -282,7 +309,7 @@ function do_backup() {
     COMMAND="/tmp/backup_postgresql.sh ${BACKUP_ARGS[$DB_LOC]}"
   fi
 
-  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- $COMMAND
+  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- $COMMAND
 
   unlock
 }
@@ -301,7 +328,7 @@ function do_list_archives() {
   ensure_ondemand_pod_exists
 
   # Execute the command in the on-demand pod
-  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- /tmp/restore_postgresql.sh list_archives "$LOCATION"
+  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- /tmp/restore_postgresql.sh list_archives "$LOCATION"
 }
 
 # Params: [-rp] <archive>
@@ -333,7 +360,7 @@ function do_list_databases() {
   ensure_ondemand_pod_exists
 
   # Execute the command in the on-demand pod
-  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- /tmp/restore_postgresql.sh list_databases "$ARCHIVE" "$LOCATION"
+  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- /tmp/restore_postgresql.sh list_databases "$ARCHIVE" "$LOCATION"
 }
 
 # Params: [-rp] <archive> <database>
@@ -367,7 +394,7 @@ function do_list_tables() {
   ensure_ondemand_pod_exists
 
   # Execute the command in the on-demand pod
-  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- /tmp/restore_postgresql.sh list_tables "$ARCHIVE" "$DATABASE" "$LOCATION"
+  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- /tmp/restore_postgresql.sh list_tables "$ARCHIVE" "$DATABASE" "$LOCATION"
 }
 
 # Params: [-rp] <archive> <database> <table>
@@ -402,7 +429,7 @@ function do_list_rows() {
   ensure_ondemand_pod_exists
 
   # Execute the command in the on-demand pod
-  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- /tmp/restore_postgresql.sh list_rows "$ARCHIVE" "$DATABASE" "$TABLE" "$LOCATION"
+  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- /tmp/restore_postgresql.sh list_rows "$ARCHIVE" "$DATABASE" "$TABLE" "$LOCATION"
 }
 
 # Params: [-rp] <archive> <database> <table>
@@ -437,7 +464,7 @@ function do_list_schema() {
   ensure_ondemand_pod_exists
 
   # Execute the command in the on-demand pod
-  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- /tmp/restore_postgresql.sh list_schema "$ARCHIVE" "$DATABASE" "$TABLE" "$LOCATION"
+  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- /tmp/restore_postgresql.sh list_schema "$ARCHIVE" "$DATABASE" "$TABLE" "$LOCATION"
 }
 
 # Params: [-rp] <archive>
@@ -469,7 +496,7 @@ function do_delete_archive() {
   ensure_ondemand_pod_exists
 
   # Execute the command in the on-demand pod
-  kubectl exec -i -n "${NAMESPACE}" "${ONDEMAND_POD}" -- /tmp/restore_postgresql.sh delete_archive "${ARCHIVE}" "${LOCATION}"
+  kubectl exec -i -n "${NAMESPACE}" "${ONDEMAND_POD}" -c "${ONDEMAND_JOB}" -- /tmp/restore_postgresql.sh delete_archive "${ARCHIVE}" "${LOCATION}"
 }
 
 # Params: [namespace]
@@ -742,7 +769,7 @@ function do_restore() {
   lock 60
 
   # Execute the command in the on-demand pod
-  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- /tmp/restore_postgresql.sh restore "$ARCHIVE" "$DATABASE" "$LOCATION"
+  kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- /tmp/restore_postgresql.sh restore "$ARCHIVE" "$DATABASE" "$LOCATION"
 
   unlock
 }
diff --git a/charts/postgresql-utility/templates/bin/utility/_pg_ondemand_job.sh.tpl b/charts/postgresql-utility/templates/bin/utility/_pg_ondemand_job.sh.tpl
index f5a4eba0..68de0e62 100644
--- a/charts/postgresql-utility/templates/bin/utility/_pg_ondemand_job.sh.tpl
+++ b/charts/postgresql-utility/templates/bin/utility/_pg_ondemand_job.sh.tpl
@@ -69,7 +69,7 @@ spec:
 {{ dict "envAll" $envAll "application" "postgresql_ondemand" "container" "postgresql_ondemand" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
           command:
             - /bin/sleep
-            - "1000000"
+            - "{{ .Values.conf.postgresql_ondemand.ondemapd_pod_sleep_time }}"
           env:
             - name: POSTGRESQL_ADMIN_USER
               valueFrom:
diff --git a/charts/postgresql-utility/values.yaml b/charts/postgresql-utility/values.yaml
index 788d3477..7ddee626 100644
--- a/charts/postgresql-utility/values.yaml
+++ b/charts/postgresql-utility/values.yaml
@@ -144,6 +144,8 @@ conf:
       psql: CommandFilter, psql, root
       kubectl: CommandFilter, kubectl, root
       dbutils: CommandFilter, dbutils, nobody
+  postgresql_ondemand:
+    ondemapd_pod_sleep_time: 3600
   postgresqlrootwrap:
     DEFAULT:
       # Configuration for postgresql-rootwrap