From c6436ce2c934ded1bec6cb5a9107a7ad1b3ecfee Mon Sep 17 00:00:00 2001 From: Sergiy Markin Date: Sat, 11 Mar 2023 20:19:47 +0000 Subject: [PATCH] [dbutils] Wait for ON_DEMAND_POD to be created This PS is intended to fix intermittent issue on some sites when database backup ondemand job fails to detect ON_DEMAND_POD when it is still creating. Change-Id: I4940a5f5a4d0739d8ccc9e55d042dfc22884e789 --- charts/etcdctl-utility/Chart.yaml | 2 +- .../templates/bin/utility/_dbutils.tpl | 14 +++++++++++++- charts/mysqlclient-utility/Chart.yaml | 2 +- .../templates/bin/utility/_dbutils.tpl | 14 +++++++++++++- charts/postgresql-utility/Chart.yaml | 2 +- .../templates/bin/utility/_dbutils.tpl | 14 +++++++++++++- images/mysqlclient-utility/Dockerfile.ubuntu_focal | 2 +- 7 files changed, 43 insertions(+), 7 deletions(-) diff --git a/charts/etcdctl-utility/Chart.yaml b/charts/etcdctl-utility/Chart.yaml index 0c2d0b3d..729d2090 100644 --- a/charts/etcdctl-utility/Chart.yaml +++ b/charts/etcdctl-utility/Chart.yaml @@ -13,4 +13,4 @@ apiVersion: v1 description: etcdctl Client name: etcdctl-utility -version: 0.1.2 +version: 0.1.3 diff --git a/charts/etcdctl-utility/templates/bin/utility/_dbutils.tpl b/charts/etcdctl-utility/templates/bin/utility/_dbutils.tpl index f4cedbe2..018865fb 100755 --- a/charts/etcdctl-utility/templates/bin/utility/_dbutils.tpl +++ b/charts/etcdctl-utility/templates/bin/utility/_dbutils.tpl @@ -119,7 +119,19 @@ function ensure_ondemand_pod_exists() { exit 1 fi - sleep 10s + # waiting for ondemand pod to be created + RETRIES=10 + until kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" | grep ondemand; do + RETRIES=$((RETRIES-1)) + if [ ${RETRIES} -ge 1 ]; then + echo "ONDEMAND_POD is being created... Waiting for 10 seconds... Retries left ${RETRIES}..." + sleep 10s + else + echo "ERROR: Failed to create a new on-demand pod. Exiting..." + exit 1 + fi + done + ONDEMAND_POD=$(kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" -o json | jq -r .items[].metadata.name) if [[ -z "$ONDEMAND_POD" ]]; then echo "ERROR: Failed to obtain the ONDEMAND_POD name." diff --git a/charts/mysqlclient-utility/Chart.yaml b/charts/mysqlclient-utility/Chart.yaml index 9934b989..352c44e1 100644 --- a/charts/mysqlclient-utility/Chart.yaml +++ b/charts/mysqlclient-utility/Chart.yaml @@ -13,4 +13,4 @@ apiVersion: v1 description: Porthole MySql Client name: mysqlclient-utility -version: 0.1.2 +version: 0.1.3 diff --git a/charts/mysqlclient-utility/templates/bin/utility/_dbutils.tpl b/charts/mysqlclient-utility/templates/bin/utility/_dbutils.tpl index 8a68d479..befbbbac 100755 --- a/charts/mysqlclient-utility/templates/bin/utility/_dbutils.tpl +++ b/charts/mysqlclient-utility/templates/bin/utility/_dbutils.tpl @@ -133,7 +133,19 @@ function ensure_ondemand_pod_exists() { exit 1 fi - sleep 10s + # waiting for ondemand pod to be created + RETRIES=10 + until kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" | grep ondemand; do + RETRIES=$((RETRIES-1)) + if [ ${RETRIES} -ge 1 ]; then + echo "ONDEMAND_POD is being created... Waiting for 10 seconds... Retries left ${RETRIES}..." + sleep 10s + else + echo "ERROR: Failed to create a new on-demand pod. Exiting..." + exit 1 + fi + done + ONDEMAND_POD=$(kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" -o json | jq -r .items[].metadata.name) if [[ -z "$ONDEMAND_POD" ]]; then echo "ERROR: Failed to obtain the ONDEMAND_POD name." diff --git a/charts/postgresql-utility/Chart.yaml b/charts/postgresql-utility/Chart.yaml index 87ba2a02..b7012790 100644 --- a/charts/postgresql-utility/Chart.yaml +++ b/charts/postgresql-utility/Chart.yaml @@ -14,4 +14,4 @@ apiVersion: v1 description: PostgreSQL Client name: postgresql-utility -version: 0.1.1 +version: 0.1.2 diff --git a/charts/postgresql-utility/templates/bin/utility/_dbutils.tpl b/charts/postgresql-utility/templates/bin/utility/_dbutils.tpl index 2e4dcded..03c7bc0d 100755 --- a/charts/postgresql-utility/templates/bin/utility/_dbutils.tpl +++ b/charts/postgresql-utility/templates/bin/utility/_dbutils.tpl @@ -133,7 +133,19 @@ function ensure_ondemand_pod_exists() { exit 1 fi - sleep 10s + # waiting for ondemand pod to be created + RETRIES=10 + until kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" | grep ondemand; do + RETRIES=$((RETRIES-1)) + if [ ${RETRIES} -ge 1 ]; then + echo "ONDEMAND_POD is being created... Waiting for 10 seconds... Retries left ${RETRIES}..." + sleep 10s + else + echo "ERROR: Failed to create a new on-demand pod. Exiting..." + exit 1 + fi + done + ONDEMAND_POD=$(kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" -o json | jq -r .items[].metadata.name) if [[ -z "$ONDEMAND_POD" ]]; then echo "ERROR: Failed to obtain the ONDEMAND_POD name." diff --git a/images/mysqlclient-utility/Dockerfile.ubuntu_focal b/images/mysqlclient-utility/Dockerfile.ubuntu_focal index b1111e6f..d5f9149c 100644 --- a/images/mysqlclient-utility/Dockerfile.ubuntu_focal +++ b/images/mysqlclient-utility/Dockerfile.ubuntu_focal @@ -46,7 +46,7 @@ RUN set -xe \ && apt-get clean -y \ && curl --silent -L https://dl.k8s.io/v${KUBE_VERSION}/kubernetes-client-linux-amd64.tar.gz \ | tar -zC /usr/bin --strip-components=3 --wildcards -x "*/*/*/kubectl" \ - && curl --silent -L https://percona.com/get/percona-toolkit.tar.gz \ + && curl --silent -L https://downloads.percona.com/downloads/percona-toolkit/3.5.1/binary/tarball/percona-toolkit-3.5.1_x86_64.tar.gz \ | tar -zC /usr/local/bin/ --strip-components=2 --wildcards -x "*/*/pt-show-grants" \ && rm -rf \ /tmp/* \