airship/porthole
Code Issues Proposed changes

[utility] Ondemand jobs improvements

Browse Source 
This PS is intended to address several issues related to ondemand pod
management having the goal to improve testing process:

- replaces 1 million seconds delay in main container with
  configurable value set by default to 3,600 seconds
- improvement in dbutils.sh script that cerated ondemand job and waits
  for ondemand pod creation

The changes are affecting etcdctl-utility, mysqlclient-utility and
postgresql-utility charts.

Change-Id: I9a2f2c6d2e84414cbb3e175c0b94f9780e8a6388
This commit is contained in:
Sergiy Markin 2023-12-05 21:41:19 +00:00
parent 7af0a01b89
commit f0efd75c99
12 changed files with 158 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
charts/etcdctl-utility/Chart.yaml
View File

@ -13,4 +13,4 @@
apiVersion: v1
description: etcdctl Client
name: etcdctl-utility
version: 0.1.3
version: 0.1.4

65
charts/etcdctl-utility/templates/bin/utility/_dbutils.tpl
View File

@ -78,19 +78,20 @@ function check_args() {
function ensure_ondemand_pod_exists() {
# Determine the status of the on demand pod if it exists
POD_LISTING=$(kubectl get pod -n "$NAMESPACE" | grep "$ONDEMAND_JOB")
POD_LISTING=$(kubectl get pod --selector application="$ONDEMAND_JOB",component=ondemand --no-headers --namespace "$NAMESPACE" 2>/dev/null)
unset STATUS
unset ONDEMAND_POD
if [[ ! -z "$POD_LISTING" ]]; then
ONDEMAND_POD=$(echo "$POD_LISTING" | awk '{print $1}')
STATUS=$(echo "$POD_LISTING" | awk '{print $3}')
if [[ "$STATUS" == "Terminating" ]]; then
kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=30s
kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s
unset ONDEMAND_POD
elif [[ "$STATUS" != "Running" ]]; then
kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=30s
unset STATUS
fi
fi
POD_LISTING=$(kubectl get pod -n "$NAMESPACE" | grep "$ONDEMAND_JOB")
POD_LISTING=$(kubectl get pod --selector application="$ONDEMAND_JOB",component=ondemand --no-headers --namespace "$NAMESPACE" 2>/dev/null)
if [[ ! -z "$POD_LISTING" ]]; then
STATUS=$(echo "$POD_LISTING" | awk '{print $3}')
CONTAINERS=$(echo "$POD_LISTING" | awk '{print $2}')
@ -98,14 +99,16 @@ function ensure_ondemand_pod_exists() {
# we find any which are not ready remove them, even if completed.
if [[ $STATUS != "Running" || $CONTAINERS != "1/1" ]]; then
echo "Found an old on-demand pod; removing it."
remove_job "$NAMESPACE" "$ONDEMAND_JOB"
remove_job "$ONDEMAND_JOB"
if [[ $? -ne 0 ]]; then
echo "ERROR: Failed to remove old on-demand pod. Exiting..."
exit 1
fi
unset ONDEMAND_POD
unset STATUS
else
# Pod is already running and ready
ONDEMAND_POD=$(kubectl get pod -n "$NAMESPACE" | grep "$ONDEMAND_JOB" | awk '{print $1}')
ONDEMAND_POD=$(kubectl get pod --selector application="$ONDEMAND_JOB",component=ondemand --no-headers --namespace "$NAMESPACE" | awk '{print $1}')
fi
fi
@ -119,19 +122,38 @@ function ensure_ondemand_pod_exists() {
exit 1
fi
# waiting for ondemand pod to be created
# waiting for ondemand job to be created
RETRIES=10
until kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" | grep ondemand; do
until kubectl get job -n "$NAMESPACE" --no-headers | grep "$ONDEMAND_JOB"; do
RETRIES=$((RETRIES-1))
if [ ${RETRIES} -ge 1 ]; then
echo "ONDEMAND_POD is being created... Waiting for 10 seconds... Retries left ${RETRIES}..."
echo "ONDEMAND_JOB is being created... Waiting for 10 seconds... Retries left ${RETRIES}..."
sleep 10s
else
echo "ERROR: Failed to create a new on-demand pod. Exiting..."
echo "ERROR: Failed to create a new on-demand job. Exiting..."
exit 1
fi
done
# Determine the status of the on demand pod if it exists
POD_LISTING=$(kubectl get pod --selector application="$ONDEMAND_JOB",component=ondemand --no-headers --namespace "$NAMESPACE")
unset STATUS
unset ONDEMAND_POD
if [[ ! -z "$POD_LISTING" ]]; then
ONDEMAND_POD=$(echo "$POD_LISTING" | awk '{print $1}')
STATUS=$(echo "$POD_LISTING" | awk '{print $3}')
if [[ "$STATUS" == "Terminating" ]]; then
kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s
unset ONDEMAND_POD
elif [[ "$STATUS" != "Running" ]]; then
kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s
elif [[ "$STATUS" != "Pending" ]]; then
kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s
fi
fi
ONDEMAND_POD=$(kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" -o json | jq -r .items[].metadata.name)
if [[ -z "$ONDEMAND_POD" ]]; then
echo "ERROR: Failed to obtain the ONDEMAND_POD name."
@ -148,7 +170,7 @@ function ensure_ondemand_pod_exists() {
export ONDEMAND_POD
}
# Params: <job>
# Params: [namespace] <job>
function remove_job() {
JOB=$1
@ -157,13 +179,18 @@ function remove_job() {
kubectl get job -n "$NAMESPACE" "$JOB"
if [[ $? -eq 0 ]]; then
echo "Removing on-demand job $NAMESPACE $JOB"
ONDEMAND_POD=$(kubectl get pod -n "$NAMESPACE" | grep "$ONDEMAND_JOB" | awk '{print $1}')
ONDEMAND_POD=$(kubectl get pod -n "$NAMESPACE" | grep "$JOB" | awk '{print $1}')
kubectl delete job --ignore-not-found -n "$NAMESPACE" "$JOB"
kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" pod/"${ONDEMAND_POD}" &>/dev/null
kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" job/"${JOB}"
if [[ $? -ne 0 ]]; then
echo "ERROR: could not destroy the $NAMESPACE $JOB job. Exiting..."
exit 1
fi
kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" pod/"${ONDEMAND_POD}"
if [[ $? -ne 0 ]]; then
echo "ERROR: could not destroy the $NAMESPACE $ONDEMAND_POD pod. Exiting..."
exit 1
fi
fi
}
@ -181,7 +208,7 @@ function do_backup() {
ensure_ondemand_pod_exists
# Execute the command in the on-demand pod
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- /tmp/backup_etcd.sh
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- /tmp/backup_etcd.sh
}
# Params: [-rp] <node>
@ -198,7 +225,7 @@ function do_list_archives() {
ensure_ondemand_pod_exists
# Execute the command in the on-demand pod
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- /tmp/restore_etcd.sh list_archives "$LOCATION"
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- /tmp/restore_etcd.sh list_archives "$LOCATION"
}
# Params: [-rp] <archive> <anchor> <node>
@ -231,7 +258,7 @@ function do_restore() {
ensure_ondemand_pod_exists
# Execute the command in the on-demand pod
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- /tmp/restore_etcd.sh restore "$ARCHIVE" "$DATABASE" "$LOCATION"
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- /tmp/restore_etcd.sh restore "$ARCHIVE" "$DATABASE" "$LOCATION"
}
# Params: [-rp] <archive> <node>
@ -262,13 +289,13 @@ function do_delete_archive() {
ensure_ondemand_pod_exists
# Execute the command in the on-demand pod
kubectl exec -i -n "${NAMESPACE}" "${ONDEMAND_POD}" -- /tmp/restore_etcd.sh delete_archive "${ARCHIVE}" "${LOCATION}"
kubectl exec -i -n "${NAMESPACE}" "${ONDEMAND_POD}" -c "${ONDEMAND_JOB}" -- /tmp/restore_etcd.sh delete_archive "${ARCHIVE}" "${LOCATION}"
}
function do_cleanup() {
if [[ "$KEEP_POD" == "false" ]]; then
remove_job "$ONDEMAND_JOB"
remove_job "${ONDEMAND_JOB}"
unset ONDEMAND_POD

2
charts/etcdctl-utility/templates/bin/utility/_etcd_ondemand_job.sh.tpl
View File

@ -56,7 +56,7 @@ spec:
{{ dict "envAll" $envAll "application" "etcd_ondemand" "container" "etcd_ondemand" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
command:
- /bin/sleep
- "1000000"
- "{{ .Values.conf.etcd_ondemand.ondemapd_pod_sleep_time }}"
env:
- name: ETCDCTL_API
value: "{{ .Values.conf.etcd.etcdctl_api }}"

2
charts/etcdctl-utility/values.yaml
View File

@ -142,6 +142,8 @@ conf:
dbutils: CommandFilter, dbutils, nobody
etcdctl: CommandFilter, etcdctl, root
kubectl: CommandFilter, kubectl, root
etcd_ondemand:
ondemapd_pod_sleep_time: 3600
etcdctlrootwrapconf:
DEFAULT:
# Configuration for etcdctl-rootwrap

2
charts/mysqlclient-utility/Chart.yaml
View File

@ -13,4 +13,4 @@
apiVersion: v1
description: Porthole MySql Client
name: mysqlclient-utility
version: 0.1.4
version: 0.1.5

69
charts/mysqlclient-utility/templates/bin/utility/_dbutils.tpl
View File

@ -92,19 +92,20 @@ function check_args() {
function ensure_ondemand_pod_exists() {
# Determine the status of the on demand pod if it exists
POD_LISTING=$(kubectl get pod -n "$NAMESPACE" | grep "$ONDEMAND_JOB")
POD_LISTING=$(kubectl get pod --selector application="$ONDEMAND_JOB",component=ondemand --no-headers --namespace "$NAMESPACE" 2>/dev/null)
unset STATUS
unset ONDEMAND_POD
if [[ ! -z "$POD_LISTING" ]]; then
ONDEMAND_POD=$(echo "$POD_LISTING" | awk '{print $1}')
STATUS=$(echo "$POD_LISTING" | awk '{print $3}')
if [[ "$STATUS" == "Terminating" ]]; then
kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=30s
kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s
unset ONDEMAND_POD
elif [[ "$STATUS" != "Running" ]]; then
kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=30s
unset STATUS
fi
fi
POD_LISTING=$(kubectl get pod -n "$NAMESPACE" | grep "$ONDEMAND_JOB")
POD_LISTING=$(kubectl get pod --selector application="$ONDEMAND_JOB",component=ondemand --no-headers --namespace "$NAMESPACE" 2>/dev/null)
if [[ ! -z "$POD_LISTING" ]]; then
STATUS=$(echo "$POD_LISTING" | awk '{print $3}')
CONTAINERS=$(echo "$POD_LISTING" | awk '{print $2}')
@ -117,9 +118,11 @@ function ensure_ondemand_pod_exists() {
echo "ERROR: Failed to remove old on-demand pod. Exiting..."
exit 1
fi
unset ONDEMAND_POD
unset STATUS
else
# Pod is already running and ready
ONDEMAND_POD=$(kubectl get pod -n "$NAMESPACE" | grep "$ONDEMAND_JOB" | awk '{print $1}')
ONDEMAND_POD=$(kubectl get pod --selector application="$ONDEMAND_JOB",component=ondemand --no-headers --namespace "$NAMESPACE" | awk '{print $1}')
fi
fi
@ -133,19 +136,38 @@ function ensure_ondemand_pod_exists() {
exit 1
fi
# waiting for ondemand pod to be created
RETRIES=10
until kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" | grep ondemand; do
# waiting for ondemand job to be created
RETRIES=30
until kubectl get job -n "$NAMESPACE" --no-headers | grep "$ONDEMAND_JOB"; do
RETRIES=$((RETRIES-1))
if [ ${RETRIES} -ge 1 ]; then
echo "ONDEMAND_POD is being created... Waiting for 10 seconds... Retries left ${RETRIES}..."
echo "ONDEMAND_JOB is being created... Waiting for 10 seconds... Retries left ${RETRIES}..."
sleep 10s
else
echo "ERROR: Failed to create a new on-demand pod. Exiting..."
echo "ERROR: Failed to create a new on-demand job. Exiting..."
exit 1
fi
done
# Determine the status of the on demand pod if it exists
POD_LISTING=$(kubectl get pod --selector application="$ONDEMAND_JOB",component=ondemand --no-headers --namespace "$NAMESPACE")
unset STATUS
unset ONDEMAND_POD
if [[ ! -z "$POD_LISTING" ]]; then
ONDEMAND_POD=$(echo "$POD_LISTING" | awk '{print $1}')
STATUS=$(echo "$POD_LISTING" | awk '{print $3}')
if [[ "$STATUS" == "Terminating" ]]; then
kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s
unset ONDEMAND_POD
elif [[ "$STATUS" != "Running" ]]; then
kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s
elif [[ "$STATUS" != "Pending" ]]; then
kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s
fi
fi
ONDEMAND_POD=$(kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" -o json | jq -r .items[].metadata.name)
if [[ -z "$ONDEMAND_POD" ]]; then
echo "ERROR: Failed to obtain the ONDEMAND_POD name."
@ -206,7 +228,7 @@ function setup_namespace() {
return 0
}
# Params: <namespace> <job>
# Params: [namespace] <job>
function remove_job() {
NAMESPACE=$1
@ -218,11 +240,16 @@ function remove_job() {
echo "Removing on-demand job $NAMESPACE $JOB"
ONDEMAND_POD=$(kubectl get pod -n "$NAMESPACE" | grep "$JOB" | awk '{print $1}')
kubectl delete job --ignore-not-found -n "$NAMESPACE" "$JOB"
kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" pod/"${ONDEMAND_POD}" &>/dev/null
kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" job/"${JOB}"
if [[ $? -ne 0 ]]; then
echo "ERROR: could not destroy the $NAMESPACE $JOB job. Exiting..."
exit 1
fi
kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" pod/"${ONDEMAND_POD}"
if [[ $? -ne 0 ]]; then
echo "ERROR: could not destroy the $NAMESPACE $ONDEMAND_POD pod. Exiting..."
exit 1
fi
fi
}
@ -285,7 +312,7 @@ function do_backup() {
COMMAND="/tmp/backup_mariadb.sh ${BACKUP_ARGS[${DB_LOC}]}"
fi
kubectl exec -i -n "${NAMESPACE}" "${ONDEMAND_POD}" -- ${COMMAND}
kubectl exec -i -n "${NAMESPACE}" "${ONDEMAND_POD}" -c "${ONDEMAND_JOB}" -- ${COMMAND}
unlock
}
@ -304,7 +331,7 @@ function do_list_archives() {
ensure_ondemand_pod_exists
# Execute the command in the on-demand pod
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- /tmp/restore_mariadb.sh list_archives "$LOCATION"
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- /tmp/restore_mariadb.sh list_archives "$LOCATION"
}
# Params: [-rp] <archive>
@ -336,7 +363,7 @@ function do_list_databases() {
ensure_ondemand_pod_exists
# Execute the command in the on-demand pod
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- /tmp/restore_mariadb.sh list_databases "$ARCHIVE" "$LOCATION"
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- /tmp/restore_mariadb.sh list_databases "$ARCHIVE" "$LOCATION"
}
# Params: [-rp] <archive> <database>
@ -370,7 +397,7 @@ function do_list_tables() {
ensure_ondemand_pod_exists
# Execute the command in the on-demand pod
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- /tmp/restore_mariadb.sh list_tables "$ARCHIVE" "$DATABASE" "$LOCATION"
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- /tmp/restore_mariadb.sh list_tables "$ARCHIVE" "$DATABASE" "$LOCATION"
}
# Params: [-rp] <archive> <database> <table>
@ -404,7 +431,7 @@ function do_list_rows() {
ensure_ondemand_pod_exists
# Execute the command in the on-demand pod
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- /tmp/restore_mariadb.sh list_rows "$ARCHIVE" "$DATABASE" "$TABLE" "$LOCATION"
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- /tmp/restore_mariadb.sh list_rows "$ARCHIVE" "$DATABASE" "$TABLE" "$LOCATION"
}
# Params: [-rp] <archive> <database> <table>
@ -438,7 +465,7 @@ function do_list_schema() {
ensure_ondemand_pod_exists
# Execute the command in the on-demand pod
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- /tmp/restore_mariadb.sh list_schema "$ARCHIVE" "$DATABASE" "$TABLE" "$LOCATION"
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- /tmp/restore_mariadb.sh list_schema "$ARCHIVE" "$DATABASE" "$TABLE" "$LOCATION"
}
# Params: [-rp] <archive>
@ -470,7 +497,7 @@ function do_delete_archive() {
ensure_ondemand_pod_exists
# Execute the command in the on-demand pod
kubectl exec -i -n "${NAMESPACE}" "${ONDEMAND_POD}" -- /tmp/restore_mariadb.sh delete_archive "${ARCHIVE}" "${LOCATION}"
kubectl exec -i -n "${NAMESPACE}" "${ONDEMAND_POD}" -c "${ONDEMAND_JOB}" -- /tmp/restore_mariadb.sh delete_archive "${ARCHIVE}" "${LOCATION}"
}
# Params: [-p] <namespace>
@ -838,7 +865,7 @@ function do_restore() {
lock 300
# Execute the command in the on-demand pod
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- /tmp/restore_mariadb.sh restore "$ARCHIVE" "$DATABASE" "$LOCATION"
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- /tmp/restore_mariadb.sh restore "$ARCHIVE" "$DATABASE" "$LOCATION"
unlock
}

13
charts/mysqlclient-utility/templates/bin/utility/_mariadb_ondemand_job.sh.tpl
View File

@ -92,11 +92,9 @@ spec:
image: ${MYSQLCLIENT_UTILTIY_IMAGE_NAME}
{{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_ondemand | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
{{ dict "envAll" $envAll "application" "mariadb_ondemand" "container" "mariadb_ondemand" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
command: ["/bin/sh"]
args:
- -c
- >-
/bin/sleep "1000000"
command:
- /bin/sleep
- "{{ .Values.conf.mariadb_ondemand.ondemapd_pod_sleep_time }}"
env:
- name: MARIADB_BACKUP_BASE_DIR
valueFrom:
@ -246,7 +244,10 @@ if $TLS_ENABLED; then
- name: MYSQL_HISTFILE
value: /dev/null
command:
- /tmp/start_verification_server.sh
- /bin/sh
args:
- -c
- ( /tmp/start_verification_server.sh )& /bin/sleep {{ .Values.conf.mariadb_ondemand.ondemapd_pod_sleep_time }}
volumeMounts:
- name: pod-tmp
mountPath: /tmp

2
charts/mysqlclient-utility/values.yaml
View File

@ -172,6 +172,8 @@ conf:
mysql: CommandFilter, mysql, root
kubectl: CommandFilter, kubectl, root
dbutils: CommandFilter, dbutils, nobody
mariadb_ondemand:
ondemapd_pod_sleep_time: 3600
mysqlclientrootwrapconf:
DEFAULT:
# Configuration for mysqlclient-rootwrap

2
charts/postgresql-utility/Chart.yaml
View File

@ -14,4 +14,4 @@
apiVersion: v1
description: PostgreSQL Client
name: postgresql-utility
version: 0.1.2
version: 0.1.3

65
charts/postgresql-utility/templates/bin/utility/_dbutils.tpl
View File

@ -92,19 +92,20 @@ function check_args() {
function ensure_ondemand_pod_exists() {
# Determine the status of the on demand pod if it exists
POD_LISTING=$(kubectl get pod -n "$NAMESPACE" | grep "$ONDEMAND_JOB")
POD_LISTING=$(kubectl get pod --selector application="$ONDEMAND_JOB",component=ondemand --no-headers --namespace "$NAMESPACE" 2>/dev/null)
unset STATUS
unset ONDEMAND_POD
if [[ ! -z "$POD_LISTING" ]]; then
ONDEMAND_POD=$(echo "$POD_LISTING" | awk '{print $1}')
STATUS=$(echo "$POD_LISTING" | awk '{print $3}')
if [[ "$STATUS" == "Terminating" ]]; then
kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=30s
kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s
unset ONDEMAND_POD
elif [[ "$STATUS" != "Running" ]]; then
kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=30s
unset STATUS
fi
fi
POD_LISTING=$(kubectl get pod -n "$NAMESPACE" | grep "$ONDEMAND_JOB")
POD_LISTING=$(kubectl get pod --selector application="$ONDEMAND_JOB",component=ondemand --no-headers --namespace "$NAMESPACE" 2>/dev/null)
if [[ ! -z "$POD_LISTING" ]]; then
STATUS=$(echo "$POD_LISTING" | awk '{print $3}')
CONTAINERS=$(echo "$POD_LISTING" | awk '{print $2}')
@ -117,9 +118,11 @@ function ensure_ondemand_pod_exists() {
echo "ERROR: Failed to remove old on-demand pod. Exiting..."
exit 1
fi
unset ONDEMAND_POD
unset STATUS
else
# Pod is already running and ready
ONDEMAND_POD=$(kubectl get pod -n "$NAMESPACE" | grep "$ONDEMAND_JOB" | awk '{print $1}')
ONDEMAND_POD=$(kubectl get pod --selector application="$ONDEMAND_JOB",component=ondemand --no-headers --namespace "$NAMESPACE" | awk '{print $1}')
fi
fi
@ -133,19 +136,38 @@ function ensure_ondemand_pod_exists() {
exit 1
fi
# waiting for ondemand pod to be created
# waiting for ondemand job to be created
RETRIES=10
until kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" | grep ondemand; do
until kubectl get job -n "$NAMESPACE" --no-headers | grep "$ONDEMAND_JOB"; do
RETRIES=$((RETRIES-1))
if [ ${RETRIES} -ge 1 ]; then
echo "ONDEMAND_POD is being created... Waiting for 10 seconds... Retries left ${RETRIES}..."
echo "ONDEMAND_JOB is being created... Waiting for 10 seconds... Retries left ${RETRIES}..."
sleep 10s
else
echo "ERROR: Failed to create a new on-demand pod. Exiting..."
echo "ERROR: Failed to create a new on-demand job. Exiting..."
exit 1
fi
done
# Determine the status of the on demand pod if it exists
POD_LISTING=$(kubectl get pod --selector application="$ONDEMAND_JOB",component=ondemand --no-headers --namespace "$NAMESPACE")
unset STATUS
unset ONDEMAND_POD
if [[ ! -z "$POD_LISTING" ]]; then
ONDEMAND_POD=$(echo "$POD_LISTING" | awk '{print $1}')
STATUS=$(echo "$POD_LISTING" | awk '{print $3}')
if [[ "$STATUS" == "Terminating" ]]; then
kubectl wait -n "$NAMESPACE" --for=delete pod/"$ONDEMAND_POD" --timeout=300s
unset ONDEMAND_POD
elif [[ "$STATUS" != "Running" ]]; then
kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s
elif [[ "$STATUS" != "Pending" ]]; then
kubectl wait -n "$NAMESPACE" --for condition=ready pod/"$ONDEMAND_POD" --timeout=300s
fi
fi
ONDEMAND_POD=$(kubectl get pods -n "$NAMESPACE" --selector=job-name="$ONDEMAND_JOB" -o json | jq -r .items[].metadata.name)
if [[ -z "$ONDEMAND_POD" ]]; then
echo "ERROR: Failed to obtain the ONDEMAND_POD name."
@ -217,11 +239,16 @@ function remove_job() {
echo "Removing on-demand job $NAMESPACE $JOB"
ONDEMAND_POD=$(kubectl get pod -n "$NAMESPACE" | grep "$JOB" | awk '{print $1}')
kubectl delete job --ignore-not-found -n "$NAMESPACE" "$JOB"
kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" pod/"${ONDEMAND_POD}" &>/dev/null
kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" job/"${JOB}"
if [[ $? -ne 0 ]]; then
echo "ERROR: could not destroy the $NAMESPACE $JOB job. Exiting..."
exit 1
fi
kubectl wait --for=delete --timeout=300s -n "$NAMESPACE" pod/"${ONDEMAND_POD}"
if [[ $? -ne 0 ]]; then
echo "ERROR: could not destroy the $NAMESPACE $ONDEMAND_POD pod. Exiting..."
exit 1
fi
fi
}
@ -282,7 +309,7 @@ function do_backup() {
COMMAND="/tmp/backup_postgresql.sh ${BACKUP_ARGS[$DB_LOC]}"
fi
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- $COMMAND
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- $COMMAND
unlock
}
@ -301,7 +328,7 @@ function do_list_archives() {
ensure_ondemand_pod_exists
# Execute the command in the on-demand pod
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- /tmp/restore_postgresql.sh list_archives "$LOCATION"
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- /tmp/restore_postgresql.sh list_archives "$LOCATION"
}
# Params: [-rp] <archive>
@ -333,7 +360,7 @@ function do_list_databases() {
ensure_ondemand_pod_exists
# Execute the command in the on-demand pod
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- /tmp/restore_postgresql.sh list_databases "$ARCHIVE" "$LOCATION"
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- /tmp/restore_postgresql.sh list_databases "$ARCHIVE" "$LOCATION"
}
# Params: [-rp] <archive> <database>
@ -367,7 +394,7 @@ function do_list_tables() {
ensure_ondemand_pod_exists
# Execute the command in the on-demand pod
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- /tmp/restore_postgresql.sh list_tables "$ARCHIVE" "$DATABASE" "$LOCATION"
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- /tmp/restore_postgresql.sh list_tables "$ARCHIVE" "$DATABASE" "$LOCATION"
}
# Params: [-rp] <archive> <database> <table>
@ -402,7 +429,7 @@ function do_list_rows() {
ensure_ondemand_pod_exists
# Execute the command in the on-demand pod
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- /tmp/restore_postgresql.sh list_rows "$ARCHIVE" "$DATABASE" "$TABLE" "$LOCATION"
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- /tmp/restore_postgresql.sh list_rows "$ARCHIVE" "$DATABASE" "$TABLE" "$LOCATION"
}
# Params: [-rp] <archive> <database> <table>
@ -437,7 +464,7 @@ function do_list_schema() {
ensure_ondemand_pod_exists
# Execute the command in the on-demand pod
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- /tmp/restore_postgresql.sh list_schema "$ARCHIVE" "$DATABASE" "$TABLE" "$LOCATION"
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- /tmp/restore_postgresql.sh list_schema "$ARCHIVE" "$DATABASE" "$TABLE" "$LOCATION"
}
# Params: [-rp] <archive>
@ -469,7 +496,7 @@ function do_delete_archive() {
ensure_ondemand_pod_exists
# Execute the command in the on-demand pod
kubectl exec -i -n "${NAMESPACE}" "${ONDEMAND_POD}" -- /tmp/restore_postgresql.sh delete_archive "${ARCHIVE}" "${LOCATION}"
kubectl exec -i -n "${NAMESPACE}" "${ONDEMAND_POD}" -c "${ONDEMAND_JOB}" -- /tmp/restore_postgresql.sh delete_archive "${ARCHIVE}" "${LOCATION}"
}
# Params: [namespace]
@ -742,7 +769,7 @@ function do_restore() {
lock 60
# Execute the command in the on-demand pod
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -- /tmp/restore_postgresql.sh restore "$ARCHIVE" "$DATABASE" "$LOCATION"
kubectl exec -i -n "$NAMESPACE" "$ONDEMAND_POD" -c "${ONDEMAND_JOB}" -- /tmp/restore_postgresql.sh restore "$ARCHIVE" "$DATABASE" "$LOCATION"
unlock
}

2
charts/postgresql-utility/templates/bin/utility/_pg_ondemand_job.sh.tpl
View File

@ -69,7 +69,7 @@ spec:
{{ dict "envAll" $envAll "application" "postgresql_ondemand" "container" "postgresql_ondemand" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
command:
- /bin/sleep
- "1000000"
- "{{ .Values.conf.postgresql_ondemand.ondemapd_pod_sleep_time }}"
env:
- name: POSTGRESQL_ADMIN_USER
valueFrom:

2
charts/postgresql-utility/values.yaml
View File

@ -144,6 +144,8 @@ conf:
psql: CommandFilter, psql, root
kubectl: CommandFilter, kubectl, root
dbutils: CommandFilter, dbutils, nobody
postgresql_ondemand:
ondemapd_pod_sleep_time: 3600
postgresqlrootwrap:
DEFAULT:
# Configuration for postgresql-rootwrap