airship/porthole
Code Issues Proposed changes

[dbutils] Wait for ON_DEMAND_POD to be created

Browse Source 
This PS is intended to fix intermittent issue on some sites when
database backup ondemand job fails to detect ON_DEMAND_POD when it is
still creating.

Change-Id: I4940a5f5a4d0739d8ccc9e55d042dfc22884e789
This commit is contained in:
Sergiy Markin 2023-03-11 20:19:47 +00:00
parent db43383cce
commit c6436ce2c9
7 changed files with 43 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
charts/etcdctl-utility/Chart.yaml
View File

@ -13,4 +13,4 @@
apiVersion: v1
description: etcdctl Client
name: etcdctl-utility
version: 0.1.2
version: 0.1.3

14
charts/etcdctl-utility/templates/bin/utility/_dbutils.tpl
View File

@ -119,7 +119,19 @@ function ensure_ondemand_pod_exists() {
exit 1
fi
sleep 10s
# waiting for ondemand pod to be created
RETRIES=10
until kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" | grep ondemand; do
RETRIES=$((RETRIES-1))
if [ ${RETRIES} -ge 1 ]; then
echo "ONDEMAND_POD is being created... Waiting for 10 seconds... Retries left ${RETRIES}..."
sleep 10s
else
echo "ERROR: Failed to create a new on-demand pod. Exiting..."
exit 1
fi
done
ONDEMAND_POD=$(kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" -o json | jq -r .items[].metadata.name)
if [[ -z "$ONDEMAND_POD" ]]; then
echo "ERROR: Failed to obtain the ONDEMAND_POD name."

2
charts/mysqlclient-utility/Chart.yaml
View File

@ -13,4 +13,4 @@
apiVersion: v1
description: Porthole MySql Client
name: mysqlclient-utility
version: 0.1.2
version: 0.1.3

14
charts/mysqlclient-utility/templates/bin/utility/_dbutils.tpl
View File

@ -133,7 +133,19 @@ function ensure_ondemand_pod_exists() {
exit 1
fi
sleep 10s
# waiting for ondemand pod to be created
RETRIES=10
until kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" | grep ondemand; do
RETRIES=$((RETRIES-1))
if [ ${RETRIES} -ge 1 ]; then
echo "ONDEMAND_POD is being created... Waiting for 10 seconds... Retries left ${RETRIES}..."
sleep 10s
else
echo "ERROR: Failed to create a new on-demand pod. Exiting..."
exit 1
fi
done
ONDEMAND_POD=$(kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" -o json | jq -r .items[].metadata.name)
if [[ -z "$ONDEMAND_POD" ]]; then
echo "ERROR: Failed to obtain the ONDEMAND_POD name."

2
charts/postgresql-utility/Chart.yaml
View File

@ -14,4 +14,4 @@
apiVersion: v1
description: PostgreSQL Client
name: postgresql-utility
version: 0.1.1
version: 0.1.2

14
charts/postgresql-utility/templates/bin/utility/_dbutils.tpl
View File

@ -133,7 +133,19 @@ function ensure_ondemand_pod_exists() {
exit 1
fi
sleep 10s
# waiting for ondemand pod to be created
RETRIES=10
until kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" | grep ondemand; do
RETRIES=$((RETRIES-1))
if [ ${RETRIES} -ge 1 ]; then
echo "ONDEMAND_POD is being created... Waiting for 10 seconds... Retries left ${RETRIES}..."
sleep 10s
else
echo "ERROR: Failed to create a new on-demand pod. Exiting..."
exit 1
fi
done
ONDEMAND_POD=$(kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" -o json | jq -r .items[].metadata.name)
if [[ -z "$ONDEMAND_POD" ]]; then
echo "ERROR: Failed to obtain the ONDEMAND_POD name."

2
images/mysqlclient-utility/Dockerfile.ubuntu_focal
View File

@ -46,7 +46,7 @@ RUN set -xe \
&& apt-get clean -y \
&& curl --silent -L https://dl.k8s.io/v${KUBE_VERSION}/kubernetes-client-linux-amd64.tar.gz \
| tar -zC /usr/bin --strip-components=3 --wildcards -x "*/*/*/kubectl" \
&& curl --silent -L https://percona.com/get/percona-toolkit.tar.gz \
&& curl --silent -L https://downloads.percona.com/downloads/percona-toolkit/3.5.1/binary/tarball/percona-toolkit-3.5.1_x86_64.tar.gz \
| tar -zC /usr/local/bin/ --strip-components=2 --wildcards -x "*/*/pt-show-grants" \
&& rm -rf \
/tmp/* \