airship/porthole
Code Issues Proposed changes

[utility] ensure_ondemand_pod_exists improvements

Browse Source 
This PS covers the case when ondemand pod does not
exists but the ondemand job is present. In this
case ensure_ondemand_pod_exists() function calls
do_cleanup() before re-creating ondemand job.

Change-Id: Iffd28317f38eb0d3ec47886e790902b12b5e22e3
This commit is contained in:
Sergiy Markin 2024-07-09 15:32:29 +00:00
parent 4f1c04c9ef
commit f60481c86a
6 changed files with 77 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
charts/etcdctl-utility/Chart.yaml
View File

@ -13,4 +13,4 @@
apiVersion: v1 apiVersion: v1
description: etcdctl Client description: etcdctl Client
name: etcdctl-utility name: etcdctl-utility
version: 0.1.5 version: 0.1.6

25
charts/etcdctl-utility/templates/bin/utility/_dbutils.tpl
View File

@ -85,7 +85,7 @@ function ensure_ondemand_pod_exists() {
ONDEMAND_POD=$(echo "$POD_LISTING" | awk '{print $1}') ONDEMAND_POD=$(echo "$POD_LISTING" | awk '{print $1}')
STATUS=$(echo "$POD_LISTING" | awk '{print $3}') STATUS=$(echo "$POD_LISTING" | awk '{print $3}')
if [[ "$STATUS" == "Terminating" ]]; then if [[ "$STATUS" == "Terminating" ]]; then
kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s 2>/dev/null
unset ONDEMAND_POD unset ONDEMAND_POD
unset STATUS unset STATUS
fi fi
@ -115,6 +115,8 @@ function ensure_ondemand_pod_exists() {
# If we reached this point with no ONDEMAND_POD, then we need to create # If we reached this point with no ONDEMAND_POD, then we need to create
# a new on-demand job. # a new on-demand job.
if [[ -z "$ONDEMAND_POD" ]]; then if [[ -z "$ONDEMAND_POD" ]]; then
echo "Cleaning up any remainings of previous runs..."
do_cleanup "$ONDEMAND_JOB" "$NAMESPACE"
echo "Creating new on-demand job in the $NAMESPACE namespace..." echo "Creating new on-demand job in the $NAMESPACE namespace..."
/tmp/etcd-ondemand-job.sh "$NAMESPACE" /tmp/etcd-ondemand-job.sh "$NAMESPACE"
if [[ $? -ne 0 ]]; then if [[ $? -ne 0 ]]; then
@ -143,12 +145,12 @@ function ensure_ondemand_pod_exists() {
ONDEMAND_POD=$(echo "$POD_LISTING" | awk '{print $1}') ONDEMAND_POD=$(echo "$POD_LISTING" | awk '{print $1}')
STATUS=$(echo "$POD_LISTING" | awk '{print $3}') STATUS=$(echo "$POD_LISTING" | awk '{print $3}')
if [[ "$STATUS" == "Terminating" ]]; then if [[ "$STATUS" == "Terminating" ]]; then
kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s 2>/dev/null
unset ONDEMAND_POD unset ONDEMAND_POD
elif [[ "$STATUS" != "Running" ]]; then elif [[ "$STATUS" != "Running" ]]; then
kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s 2>/dev/null
elif [[ "$STATUS" != "Pending" ]]; then elif [[ "$STATUS" != "Pending" ]]; then
kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s 2>/dev/null
fi fi
fi fi
@ -160,7 +162,7 @@ function ensure_ondemand_pod_exists() {
exit 1 exit 1
fi fi
kubectl wait --for condition=ready --timeout=300s -n "$NAMESPACE" "pod/${ONDEMAND_POD}" kubectl wait --for condition=ready --timeout=300s -n "$NAMESPACE" "pod/${ONDEMAND_POD}" 2>/dev/null
if [[ $? -ne 0 ]]; then if [[ $? -ne 0 ]]; then
echo "ERROR: Failed to create a new on-demand pod. Exiting..." echo "ERROR: Failed to create a new on-demand pod. Exiting..."
exit 1 exit 1
@ -176,22 +178,27 @@ function remove_job() {
JOB=$1 JOB=$1
# Cleanup the last attempted job if there is one, wait for the pod to be deleted. # Cleanup the last attempted job if there is one, wait for the pod to be deleted.
kubectl get job -n "$NAMESPACE" "$JOB" kubectl get job -n "$NAMESPACE" "$JOB" 2>/dev/null
if [[ $? -eq 0 ]]; then if [[ $? -eq 0 ]]; then
echo "Removing on-demand job $NAMESPACE $JOB" echo "Removing on-demand job $NAMESPACE $JOB"
ONDEMAND_POD=$(kubectl get pod -n "$NAMESPACE" | grep "$JOB" | awk '{print $1}') ONDEMAND_JOB=$(kubectl get job -n "$NAMESPACE" | grep "$JOB" | awk '{print $1}')
if [[ ! -z ${ONDEMAND_JOB} ]]; then
kubectl delete job --ignore-not-found -n "$NAMESPACE" "$JOB" kubectl delete job --ignore-not-found -n "$NAMESPACE" "$JOB"
kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" job/"${JOB}" kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" job/"${JOB}" 2>/dev/null
if [[ $? -ne 0 ]]; then if [[ $? -ne 0 ]]; then
echo "ERROR: could not destroy the $NAMESPACE $JOB job. Exiting..." echo "ERROR: could not destroy the $NAMESPACE $JOB job. Exiting..."
exit 1 exit 1
fi fi
kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" pod/"${ONDEMAND_POD}" fi
ONDEMAND_POD=$(kubectl get pod -n "$NAMESPACE" | grep "$JOB" | awk '{print $1}')
if [[ ! -z ${ONDEMAND_POD} ]]; then
kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" pod/"${ONDEMAND_POD}" 2>/dev/null
if [[ $? -ne 0 ]]; then if [[ $? -ne 0 ]]; then
echo "ERROR: could not destroy the $NAMESPACE $ONDEMAND_POD pod. Exiting..." echo "ERROR: could not destroy the $NAMESPACE $ONDEMAND_POD pod. Exiting..."
exit 1 exit 1
fi fi
fi fi
fi
} }
# Params: [-p] <node> # Params: [-p] <node>

2
charts/mysqlclient-utility/Chart.yaml
View File

@ -13,4 +13,4 @@
apiVersion: v1 apiVersion: v1
description: Porthole MySql Client description: Porthole MySql Client
name: mysqlclient-utility name: mysqlclient-utility
version: 0.1.8 version: 0.1.9

31
charts/mysqlclient-utility/templates/bin/utility/_dbutils.tpl
View File

@ -99,7 +99,7 @@ function ensure_ondemand_pod_exists() {
ONDEMAND_POD=$(echo "$POD_LISTING" | awk '{print $1}') ONDEMAND_POD=$(echo "$POD_LISTING" | awk '{print $1}')
STATUS=$(echo "$POD_LISTING" | awk '{print $3}') STATUS=$(echo "$POD_LISTING" | awk '{print $3}')
if [[ "$STATUS" == "Terminating" ]]; then if [[ "$STATUS" == "Terminating" ]]; then
kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s 2>/dev/null
unset ONDEMAND_POD unset ONDEMAND_POD
unset STATUS unset STATUS
fi fi
@ -129,6 +129,8 @@ function ensure_ondemand_pod_exists() {
# If we reached this point with no ONDEMAND_POD, then we need to create # If we reached this point with no ONDEMAND_POD, then we need to create
# a new on-demand job. # a new on-demand job.
if [[ -z "$ONDEMAND_POD" ]]; then if [[ -z "$ONDEMAND_POD" ]]; then
echo "Cleaning up any remainings of previous runs..."
do_cleanup "$ONDEMAND_JOB" "$NAMESPACE"
echo "Creating new on-demand job in the $NAMESPACE namespace..." echo "Creating new on-demand job in the $NAMESPACE namespace..."
/tmp/mariadb-ondemand-job.sh "$NAMESPACE" /tmp/mariadb-ondemand-job.sh "$NAMESPACE"
if [[ $? -ne 0 ]]; then if [[ $? -ne 0 ]]; then
@ -149,32 +151,31 @@ function ensure_ondemand_pod_exists() {
fi fi
done done
# Determine the status of the on demand pod if it exists # Determine the status of the on demand pod if it exists POD_LISTING=$(kubectl get pod --selector application="$ONDEMAND_JOB",component=ondemand --no-headers --namespace "$NAMESPACE")
POD_LISTING=$(kubectl get pod --selector application="$ONDEMAND_JOB",component=ondemand --no-headers --namespace "$NAMESPACE")
unset STATUS unset STATUS
unset ONDEMAND_POD unset ONDEMAND_POD
if [[ ! -z "$POD_LISTING" ]]; then if [[ ! -z "$POD_LISTING" ]]; then
ONDEMAND_POD=$(echo "$POD_LISTING" | awk '{print $1}') ONDEMAND_POD=$(echo "$POD_LISTING" | awk '{print $1}')
STATUS=$(echo "$POD_LISTING" | awk '{print $3}') STATUS=$(echo "$POD_LISTING" | awk '{print $3}')
if [[ "$STATUS" == "Terminating" ]]; then if [[ "$STATUS" == "Terminating" ]]; then
kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s 2>/dev/null
unset ONDEMAND_POD unset ONDEMAND_POD
elif [[ "$STATUS" != "Running" ]]; then elif [[ "$STATUS" != "Running" ]]; then
kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s 2>/dev/null
elif [[ "$STATUS" != "Pending" ]]; then elif [[ "$STATUS" != "Pending" ]]; then
kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s 2>/dev/null
fi fi
fi fi
ONDEMAND_POD=$(kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" -o json | jq -r .items[].metadata.name) ONDEMAND_POD=$(kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" -o json 2>/dev/null| jq -r .items[].metadata.name)
if [[ -z "$ONDEMAND_POD" ]]; then if [[ -z "$ONDEMAND_POD" ]]; then
echo "ERROR: Failed to obtain the ONDEMAND_POD name." echo "ERROR: Failed to obtain the ONDEMAND_POD name."
exit 1 exit 1
fi fi
kubectl wait --for condition=ready --timeout=300s -n "$NAMESPACE" "pod/${ONDEMAND_POD}" kubectl wait --for condition=ready --timeout=300s -n "$NAMESPACE" "pod/${ONDEMAND_POD}" 2>/dev/null
if [[ $? -ne 0 ]]; then if [[ $? -ne 0 ]]; then
echo "ERROR: Failed to create a new on-demand pod. Exiting..." echo "ERROR: Failed to create a new on-demand pod. Exiting..."
exit 1 exit 1
@ -235,22 +236,27 @@ function remove_job() {
JOB=$2 JOB=$2
# Cleanup the last attempted job if there is one, wait for the pod to be deleted. # Cleanup the last attempted job if there is one, wait for the pod to be deleted.
kubectl get job -n "$NAMESPACE" "$JOB" kubectl get job -n "$NAMESPACE" "$JOB" 2>/dev/null
if [[ $? -eq 0 ]]; then if [[ $? -eq 0 ]]; then
echo "Removing on-demand job $NAMESPACE $JOB" echo "Removing on-demand job $NAMESPACE $JOB"
ONDEMAND_POD=$(kubectl get pod -n "$NAMESPACE" | grep "$JOB" | awk '{print $1}') ONDEMAND_JOB=$(kubectl get job -n "$NAMESPACE" | grep "$JOB" | awk '{print $1}')
if [[ ! -z ${ONDEMAND_JOB} ]]; then
kubectl delete job --ignore-not-found -n "$NAMESPACE" "$JOB" kubectl delete job --ignore-not-found -n "$NAMESPACE" "$JOB"
kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" job/"${JOB}" kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" job/"${JOB}" 2>/dev/null
if [[ $? -ne 0 ]]; then if [[ $? -ne 0 ]]; then
echo "ERROR: could not destroy the $NAMESPACE $JOB job. Exiting..." echo "ERROR: could not destroy the $NAMESPACE $JOB job. Exiting..."
exit 1 exit 1
fi fi
kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" pod/"${ONDEMAND_POD}" ONDEMAND_POD=$(kubectl get pod -n "$NAMESPACE" | grep "$JOB" | awk '{print $1}')
if [[ ! -z ${ONDEMAND_POD} ]]; then
kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" pod/"${ONDEMAND_POD}" 2>/dev/null
if [[ $? -ne 0 ]]; then if [[ $? -ne 0 ]]; then
echo "ERROR: could not destroy the $NAMESPACE $ONDEMAND_POD pod. Exiting..." echo "ERROR: could not destroy the $NAMESPACE $ONDEMAND_POD pod. Exiting..."
exit 1 exit 1
fi fi
fi fi
fi
fi
} }
# Params: timeout (how long in seconds flock waited on the lock to be released) # Params: timeout (how long in seconds flock waited on the lock to be released)
@ -897,6 +903,7 @@ function do_sql_prompt() {
# Params: [namespace] # Params: [namespace]
function do_cleanup() { function do_cleanup() {
# If a namespace is given go ahead and try to clean it up. # If a namespace is given go ahead and try to clean it up.
if [[ ! -z "$2" ]]; then if [[ ! -z "$2" ]]; then
remove_job "$2" "$ONDEMAND_JOB" remove_job "$2" "$ONDEMAND_JOB"

2
charts/postgresql-utility/Chart.yaml
View File

@ -14,4 +14,4 @@
apiVersion: v1 apiVersion: v1
description: PostgreSQL Client description: PostgreSQL Client
name: postgresql-utility name: postgresql-utility
version: 0.1.4 version: 0.1.5

27
charts/postgresql-utility/templates/bin/utility/_dbutils.tpl
View File

@ -99,7 +99,7 @@ function ensure_ondemand_pod_exists() {
ONDEMAND_POD=$(echo "$POD_LISTING" | awk '{print $1}') ONDEMAND_POD=$(echo "$POD_LISTING" | awk '{print $1}')
STATUS=$(echo "$POD_LISTING" | awk '{print $3}') STATUS=$(echo "$POD_LISTING" | awk '{print $3}')
if [[ "$STATUS" == "Terminating" ]]; then if [[ "$STATUS" == "Terminating" ]]; then
kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s 2>/dev/null
unset ONDEMAND_POD unset ONDEMAND_POD
unset STATUS unset STATUS
fi fi
@ -129,6 +129,8 @@ function ensure_ondemand_pod_exists() {
# If we reached this point with no ONDEMAND_POD, then we need to create # If we reached this point with no ONDEMAND_POD, then we need to create
# a new on-demand job. # a new on-demand job.
if [[ -z "$ONDEMAND_POD" ]]; then if [[ -z "$ONDEMAND_POD" ]]; then
echo "Cleaning up any remainings of previous runs..."
do_cleanup "$ONDEMAND_JOB" "$NAMESPACE"
echo "Creating new on-demand job in the $NAMESPACE namespace..." echo "Creating new on-demand job in the $NAMESPACE namespace..."
/tmp/pg-ondemand-job.sh "$NAMESPACE" /tmp/pg-ondemand-job.sh "$NAMESPACE"
if [[ $? -ne 0 ]]; then if [[ $? -ne 0 ]]; then
@ -157,24 +159,24 @@ function ensure_ondemand_pod_exists() {
ONDEMAND_POD=$(echo "$POD_LISTING" | awk '{print $1}') ONDEMAND_POD=$(echo "$POD_LISTING" | awk '{print $1}')
STATUS=$(echo "$POD_LISTING" | awk '{print $3}') STATUS=$(echo "$POD_LISTING" | awk '{print $3}')
if [[ "$STATUS" == "Terminating" ]]; then if [[ "$STATUS" == "Terminating" ]]; then
kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s 2>/dev/null
unset ONDEMAND_POD unset ONDEMAND_POD
elif [[ "$STATUS" != "Running" ]]; then elif [[ "$STATUS" != "Running" ]]; then
kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s 2>/dev/null
elif [[ "$STATUS" != "Pending" ]]; then elif [[ "$STATUS" != "Pending" ]]; then
kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s 2>/dev/null
fi fi
fi fi
ONDEMAND_POD=$(kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" -o json | jq -r .items[].metadata.name) ONDEMAND_POD=$(kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" -o json 2>/dev/null| jq -r .items[].metadata.name)
if [[ -z "$ONDEMAND_POD" ]]; then if [[ -z "$ONDEMAND_POD" ]]; then
echo "ERROR: Failed to obtain the ONDEMAND_POD name." echo "ERROR: Failed to obtain the ONDEMAND_POD name."
exit 1 exit 1
fi fi
kubectl wait --for condition=ready --timeout=300s -n "$NAMESPACE" "pod/${ONDEMAND_POD}" kubectl wait --for condition=ready --timeout=300s -n "$NAMESPACE" "pod/${ONDEMAND_POD}" 2>/dev/null
if [[ $? -ne 0 ]]; then if [[ $? -ne 0 ]]; then
echo "ERROR: Failed to create a new on-demand pod. Exiting..." echo "ERROR: Failed to create a new on-demand pod. Exiting..."
exit 1 exit 1
@ -234,22 +236,27 @@ function remove_job() {
JOB=$2 JOB=$2
# Cleanup the last attempted job if there is one, wait for the pod to be deleted. # Cleanup the last attempted job if there is one, wait for the pod to be deleted.
kubectl get job -n "$NAMESPACE" "$JOB" kubectl get job -n "$NAMESPACE" "$JOB" 2>/dev/null
if [[ $? -eq 0 ]]; then if [[ $? -eq 0 ]]; then
echo "Removing on-demand job $NAMESPACE $JOB" echo "Removing on-demand job $NAMESPACE $JOB"
ONDEMAND_POD=$(kubectl get pod -n "$NAMESPACE" | grep "$JOB" | awk '{print $1}') ONDEMAND_JOB=$(kubectl get job -n "$NAMESPACE" | grep "$JOB" | awk '{print $1}')
if [[ ! -z ${ONDEMAND_JOB} ]]; then
kubectl delete job --ignore-not-found -n "$NAMESPACE" "$JOB" kubectl delete job --ignore-not-found -n "$NAMESPACE" "$JOB"
kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" job/"${JOB}" kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" job/"${JOB}" 2>/dev/null
if [[ $? -ne 0 ]]; then if [[ $? -ne 0 ]]; then
echo "ERROR: could not destroy the $NAMESPACE $JOB job. Exiting..." echo "ERROR: could not destroy the $NAMESPACE $JOB job. Exiting..."
exit 1 exit 1
fi fi
kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" pod/"${ONDEMAND_POD}" fi
ONDEMAND_POD=$(kubectl get pod -n "$NAMESPACE" | grep "$JOB" | awk '{print $1}')
if [[ ! -z ${ONDEMAND_POD} ]]; then
kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" pod/"${ONDEMAND_POD}" 2>/dev/null
if [[ $? -ne 0 ]]; then if [[ $? -ne 0 ]]; then
echo "ERROR: could not destroy the $NAMESPACE $ONDEMAND_POD pod. Exiting..." echo "ERROR: could not destroy the $NAMESPACE $ONDEMAND_POD pod. Exiting..."
exit 1 exit 1
fi fi
fi fi
fi
} }
# Params: timeout (how long in seconds flock waited on the lock to be released) # Params: timeout (how long in seconds flock waited on the lock to be released)