From c18ee59aff4481d7c4d7a1a074c4bd4184602bd6 Mon Sep 17 00:00:00 2001 From: Cliff Parsons Date: Thu, 6 Feb 2020 19:59:14 +0000 Subject: [PATCH] Fix postgresql database backup issue Currently postgresql database backup job will fail due to not having correct permissions on the mounted PVC. This patchset corrects the permissions on the PVC mount so that the backup pods can write to the /var/backup directory structure. Another problem was that pg_dumpall was not able to get the correct password from the admin_user.conf. This may be due to the extra lines in the file, so this patchset reads it differently in order to find the password. This was a change to the backup and restore scripts. Also there are a number of small corrections made to the error handling for both backup and restore scripts, to be consistent with the MariaDB backup/restore scripts. Change-Id: Ica361764c591099e16d03a0988f73c6976583ceb --- .../templates/bin/_backup_postgresql.sh.tpl | 57 ++++++------- .../templates/bin/_restore_postgresql.sh.tpl | 79 ++++++++++--------- .../templates/cron-job-backup-postgres.yaml | 4 +- postgresql/values.yaml | 1 - 4 files changed, 68 insertions(+), 73 deletions(-) diff --git a/postgresql/templates/bin/_backup_postgresql.sh.tpl b/postgresql/templates/bin/_backup_postgresql.sh.tpl index 9bc3f72ff..6fff8543a 100755 --- a/postgresql/templates/bin/_backup_postgresql.sh.tpl +++ b/postgresql/templates/bin/_backup_postgresql.sh.tpl @@ -14,26 +14,19 @@ # License for the specific language governing permissions and limitations # under the License. +export PGPASSWORD=$(cat /etc/postgresql/admin_user.conf \ + | grep postgres | awk -F: '{print $5}') + set -x -export PGPASSFILE=/etc/postgresql/admin_user.conf + PG_DUMPALL_OPTIONS=$POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS BACKUPS_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/current ARCHIVE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/archive -POSTGRESQL_HOST=$(cat /etc/postgresql/admin_user.conf | cut -d: -f 1) -PG_DUMPALL="pg_dumpall -U $POSTGRESQL_BACKUP_USER -h $POSTGRESQL_HOST" - -#Delete files -delete_files() { - files_to_delete=("$@") - for f in "${files_to_delete[@]}" - do - if [ -f $f ] - then - echo "Deleting file $f." - rm -rf $f - fi - done -} +LOG_FILE=/tmp/dberror.log +PG_DUMPALL="pg_dumpall \ + $POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS \ + -U $POSTGRESQL_BACKUP_USER \ + -h $POSTGRESQL_SERVICE_HOST" #Get the day delta since the archive file backup seconds_difference() { @@ -56,8 +49,7 @@ mkdir -p $BACKUPS_DIR $ARCHIVE_DIR #Dump all databases DATE=$(date +"%Y-%m-%dT%H:%M:%SZ") -pg_dumpall $POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS -U $POSTGRESQL_BACKUP_USER \ - -h $POSTGRESQL_HOST --file=$BACKUPS_DIR/postgres.all.sql 2>dberror.log +$PG_DUMPALL --file=$BACKUPS_DIR/postgres.all.sql 2>>$LOG_FILE if [[ $? -eq 0 && -s "$BACKUPS_DIR/postgres.all.sql" ]] then #Archive the current databases files @@ -73,26 +65,27 @@ then else #TODO: This can be convert into mail alert of alert send to a monitoring system echo "Backup of postgresql failed and need attention." - cat dberror.log + cat $LOG_FILE exit 1 fi #Only delete the old archive after a successful archive if [ $ARCHIVE_RET -eq 0 ] +then + if [ "$POSTGRESQL_BACKUP_DAYS_TO_KEEP" -gt 0 ] then - if [ "$POSTGRESQL_BACKUP_DAYS_TO_KEEP" -gt 0 ] + echo "Deleting backups older than $POSTGRESQL_BACKUP_DAYS_TO_KEEP days" + if [ -d $ARCHIVE_DIR ] then - echo "Deleting backups older than $POSTGRESQL_BACKUP_DAYS_TO_KEEP days" - if [ -d $ARCHIVE_DIR ] - then - for archive_file in $(ls -1 $ARCHIVE_DIR/*.gz) - do - archive_date=$( echo $archive_file | awk -F/ '{print $NF}' | cut -d'.' -f 3) - if [ "$(seconds_difference $archive_date)" -gt "$(($POSTGRESQL_BACKUP_DAYS_TO_KEEP*86400))" ] - then - rm -rf $archive_file - fi - done - fi + for archive_file in $(ls -1 $ARCHIVE_DIR/*.gz) + do + archive_date=$( echo $archive_file | awk -F/ '{print $NF}' | cut -d'.' -f 3) + if [ "$(seconds_difference $archive_date)" -gt "$(($POSTGRESQL_BACKUP_DAYS_TO_KEEP*86400))" ] + then + rm -rf $archive_file + fi + done fi + fi fi + diff --git a/postgresql/templates/bin/_restore_postgresql.sh.tpl b/postgresql/templates/bin/_restore_postgresql.sh.tpl index 2032d1f4d..43ba52af4 100755 --- a/postgresql/templates/bin/_restore_postgresql.sh.tpl +++ b/postgresql/templates/bin/_restore_postgresql.sh.tpl @@ -14,23 +14,31 @@ # License for the specific language governing permissions and limitations # under the License. -#set -x -export PGPASSFILE=/etc/postgresql/admin_user.conf +export PGPASSWORD=$(cat /etc/postgresql/admin_user.conf \ + | grep postgres | awk -F: '{print $5}') + +log_error() { + echo $1 + exit 1 +} + ARCHIVE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/archive RESTORE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/restore POSTGRESQL_HOST=$(cat /etc/postgresql/admin_user.conf | cut -d: -f 1) -LIST_OPTIONS=(list_archives list_databases) +LOG_FILE=/tmp/dbrestore.log ARGS=("$@") PSQL="psql -U $POSTGRESQL_BACKUP_USER -h $POSTGRESQL_HOST" usage() { + ret_val=$1 echo "Usage:" - echo "$0 options" + echo "Restore command options" echo "=============================" - echo "options: " + echo "help" echo "list_archives" - echo "list_databases archive_filename" - echo "restore archive_filename [DB_NAME or ALL/all]" + echo "list_databases " + echo "restore [ | ALL]" + exit $ret_val } #Delete file @@ -63,10 +71,8 @@ list_archives() { do echo $archive | cut -d '/' -f 8 done - exit 0 else - echo "Archive directory is not available." - exit 1 + log_error "Archive directory is not available." fi } @@ -122,8 +128,7 @@ restore_single_db() { single_db_name=$1 if [ -z "$single_db_name" ] then - usage - exit 1 + usage 1 fi if [ -f ${ARCHIVE_DIR}/${archive_file} ] then @@ -136,21 +141,21 @@ restore_single_db() { if [[ -f ${RESTORE_DIR}/${single_db_name}.sql && -s ${RESTORE_DIR}/${single_db_name}.sql ]] then create_db_if_not_exist $single_db_name - $PSQL -d $single_db_name -f ${RESTORE_DIR}/${single_db_name}.sql 2>dbrestore.log + $PSQL -d $single_db_name -f ${RESTORE_DIR}/${single_db_name}.sql 2>>$LOG_FILE if [ "$?" -eq 0 ] then echo "Database Restore Successful." else - echo "Database Restore Failed." + log_error "Database Restore Failed." fi else - echo "Database Dump For $single_db_name is empty or not available." + log_error "Database Dump For $single_db_name is empty or not available." fi else - echo "Database file for dump_all not available to restore from" + log_error "Database file for dump_all not available to restore from" fi else - echo "Archive does not exist" + log_error "Archive does not exist" fi } @@ -163,18 +168,18 @@ restore_all_dbs() { tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null if [ -f ${RESTORE_DIR}/postgres.all.sql ] then - $PSQL postgres -f ${RESTORE_DIR}/postgres.all.sql 2>dbrestore.log + $PSQL postgres -f ${RESTORE_DIR}/postgres.all.sql 2>>$LOG_FILE if [ "$?" -eq 0 ] then echo "Database Restore successful." else - echo "Database Restore failed." + log_error "Database Restore failed." fi else - echo "There is no database file available to restore from" + log_error "There is no database file available to restore from" fi else - echo "Archive does not exist" + log_error "Archive does not exist" fi } @@ -198,52 +203,48 @@ is_Option() { mkdir -p $RESTORE_DIR if [ ${#ARGS[@]} -gt 3 ] then - usage - exit + usage 0 elif [ ${#ARGS[@]} -eq 1 ] then - if [ $(is_Option "$LIST_OPTIONS" ${ARGS[0]}) -eq 1 ] + if [ "${ARGS[0]}" == "list_archives" ] then - ${ARGS[0]} - exit + list_archives + elif [ "${ARGS[0]}" == "help" ] + then + usage 0 else - usage - exit + usage 1 fi elif [ ${#ARGS[@]} -eq 2 ] then if [ "${ARGS[0]}" == "list_databases" ] then list_databases ${ARGS[1]} - exit 0 else - usage - exit + usage 1 fi elif [ ${#ARGS[@]} -eq 3 ] then if [ "${ARGS[0]}" != "restore" ] then - usage - exit 1 + usage 1 else if [ -f ${ARCHIVE_DIR}/${ARGS[1]} ] then #Get all the databases in that archive get_databases ${ARGS[1]} + #check if the requested database is available in the archive if [ $(is_Option "$DBS" ${ARGS[2]}) -eq 1 ] then echo "Restoring Database ${ARGS[2]} And Grants" restore_single_db ${ARGS[2]} - echo "Tail dbrestore.log for restore log." - exit 0 + echo "Tail ${LOG_FILE} for restore log." elif [ "$( echo ${ARGS[2]} | tr '[a-z]' '[A-Z]')" == "ALL" ] then echo "Restoring All The Database." restore_all_dbs - echo "Tail dbrestore.log for restore log." - exit 0 + echo "Tail ${LOG_FILE} for restore log." else echo "There is no database with that name" fi @@ -252,7 +253,7 @@ then fi fi else - usage - exit + usage 1 fi +exit 0 diff --git a/postgresql/templates/cron-job-backup-postgres.yaml b/postgresql/templates/cron-job-backup-postgres.yaml index 1014c4f84..a1d3244ed 100644 --- a/postgresql/templates/cron-job-backup-postgres.yaml +++ b/postgresql/templates/cron-job-backup-postgres.yaml @@ -41,13 +41,15 @@ spec: labels: {{ tuple $envAll "postgresql-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 12 }} spec: + securityContext: + fsGroup: 999 serviceAccountName: {{ $serviceAccountName }} restartPolicy: OnFailure nodeSelector: {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} containers: - name: postgresql-backup -{{ tuple $envAll "postgresql_backup" | include "helm-toolkit.snippets.image" | indent 14 }} +{{ tuple $envAll "postgresql" | include "helm-toolkit.snippets.image" | indent 14 }} {{ tuple $envAll $envAll.Values.pod.resources.jobs.postgresql_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }} command: - /tmp/backup_postgresql.sh diff --git a/postgresql/values.yaml b/postgresql/values.yaml index 9181412ae..bfb560040 100644 --- a/postgresql/values.yaml +++ b/postgresql/values.yaml @@ -126,7 +126,6 @@ images: image_repo_sync: docker.io/docker:17.07.0 prometheus_postgresql_exporter: docker.io/wrouesnel/postgres_exporter:v0.4.6 prometheus_postgresql_exporter_create_user: "docker.io/postgres:9.5" - postgresql_backup: "docker.io/postgres:9.5" pull_policy: "IfNotPresent" local_registry: active: false