Fix postgresql database backup issue

Currently postgresql database backup job will fail due to not having
correct permissions on the mounted PVC. This patchset corrects the
permissions on the PVC mount so that the backup pods can write to the
/var/backup directory structure.

Another problem was that pg_dumpall was not able to get the correct
password from the admin_user.conf. This may be due to the extra lines
in the file, so this patchset reads it differently in order to find
the password. This was a change to the backup and restore scripts.

Also there are a number of small corrections made to the error handling
for both backup and restore scripts, to be consistent with the MariaDB
backup/restore scripts.

Change-Id: Ica361764c591099e16d03a0988f73c6976583ceb

postgresql/templates/bin/_backup_postgresql.sh.tpl

@@ -14,26 +14,19 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+export PGPASSWORD=$(cat /etc/postgresql/admin_user.conf \
+                    | grep postgres | awk -F: '{print $5}')
+
 set -x
-export PGPASSFILE=/etc/postgresql/admin_user.conf
+
 PG_DUMPALL_OPTIONS=$POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS
 BACKUPS_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/current
 ARCHIVE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/archive
-POSTGRESQL_HOST=$(cat /etc/postgresql/admin_user.conf | cut -d: -f 1)
-PG_DUMPALL="pg_dumpall -U $POSTGRESQL_BACKUP_USER -h $POSTGRESQL_HOST"
-
-#Delete files
-delete_files() {
-  files_to_delete=("$@")
-  for f in "${files_to_delete[@]}"
-  do
-    if [ -f $f ]
-    then
-      echo "Deleting file $f."
-      rm -rf $f
-    fi
-  done
-}
+LOG_FILE=/tmp/dberror.log
+PG_DUMPALL="pg_dumpall \
+              $POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS \
+              -U $POSTGRESQL_BACKUP_USER \
+              -h $POSTGRESQL_SERVICE_HOST"
 
 #Get the day delta since the archive file backup
 seconds_difference() {
@@ -56,8 +49,7 @@ mkdir -p $BACKUPS_DIR $ARCHIVE_DIR
 
 #Dump all databases
 DATE=$(date +"%Y-%m-%dT%H:%M:%SZ")
-pg_dumpall $POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS -U $POSTGRESQL_BACKUP_USER \
-   -h  $POSTGRESQL_HOST --file=$BACKUPS_DIR/postgres.all.sql 2>dberror.log
+$PG_DUMPALL --file=$BACKUPS_DIR/postgres.all.sql 2>>$LOG_FILE
 if [[ $? -eq 0 && -s "$BACKUPS_DIR/postgres.all.sql" ]]
 then
   #Archive the current databases files
@@ -73,26 +65,27 @@ then
 else
   #TODO: This can be convert into mail alert of alert send to a monitoring system
   echo "Backup of postgresql failed and need attention."
-  cat dberror.log
+  cat $LOG_FILE
   exit 1
 fi
 
 #Only delete the old archive after a successful archive
 if [ $ARCHIVE_RET -eq 0 ]
+then
+  if [ "$POSTGRESQL_BACKUP_DAYS_TO_KEEP" -gt 0 ]
   then
-    if [ "$POSTGRESQL_BACKUP_DAYS_TO_KEEP" -gt 0 ]
+    echo "Deleting backups older than $POSTGRESQL_BACKUP_DAYS_TO_KEEP days"
+    if [ -d $ARCHIVE_DIR ]
     then
-      echo "Deleting backups older than $POSTGRESQL_BACKUP_DAYS_TO_KEEP days"
-      if [ -d $ARCHIVE_DIR ]
-      then
-        for archive_file in $(ls -1 $ARCHIVE_DIR/*.gz)
-        do
-          archive_date=$( echo $archive_file | awk -F/ '{print $NF}' | cut -d'.' -f 3)
-          if [ "$(seconds_difference $archive_date)" -gt "$(($POSTGRESQL_BACKUP_DAYS_TO_KEEP*86400))" ]
-          then
-            rm -rf $archive_file
-          fi
-        done
-      fi
+      for archive_file in $(ls -1 $ARCHIVE_DIR/*.gz)
+      do
+        archive_date=$( echo $archive_file | awk -F/ '{print $NF}' | cut -d'.' -f 3)
+        if [ "$(seconds_difference $archive_date)" -gt "$(($POSTGRESQL_BACKUP_DAYS_TO_KEEP*86400))" ]
+        then
+          rm -rf $archive_file
+        fi
+      done
     fi
+  fi
 fi
+

postgresql/templates/bin/_restore_postgresql.sh.tpl

@@ -14,23 +14,31 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-#set -x
-export PGPASSFILE=/etc/postgresql/admin_user.conf
+export PGPASSWORD=$(cat /etc/postgresql/admin_user.conf \
+                    | grep postgres | awk -F: '{print $5}')
+
+log_error() {
+  echo $1
+  exit 1
+}
+
 ARCHIVE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/archive
 RESTORE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/restore
 POSTGRESQL_HOST=$(cat /etc/postgresql/admin_user.conf | cut -d: -f 1)
-LIST_OPTIONS=(list_archives list_databases)
+LOG_FILE=/tmp/dbrestore.log
 ARGS=("$@")
 PSQL="psql -U $POSTGRESQL_BACKUP_USER -h $POSTGRESQL_HOST"
 
 usage() {
+  ret_val=$1
   echo "Usage:"
-  echo "$0 options"
+  echo "Restore command options"
   echo "============================="
-  echo "options: "
+  echo "help"
   echo "list_archives"
-  echo "list_databases archive_filename"
-  echo "restore archive_filename [DB_NAME or ALL/all]"
+  echo "list_databases <archive_filename>"
+  echo "restore <archive_filename> [<db_name> | ALL]"
+  exit $ret_val
 }
 
 #Delete file
@@ -63,10 +71,8 @@ list_archives() {
     do
       echo $archive | cut -d '/' -f 8
     done
-    exit 0
   else
-    echo "Archive directory is not available."
-    exit 1
+    log_error "Archive directory is not available."
   fi
 }
 
@@ -122,8 +128,7 @@ restore_single_db() {
   single_db_name=$1
   if [ -z "$single_db_name" ]
   then
-    usage
-    exit 1
+    usage 1
   fi
   if [ -f ${ARCHIVE_DIR}/${archive_file} ]
   then
@@ -136,21 +141,21 @@ restore_single_db() {
       if [[ -f ${RESTORE_DIR}/${single_db_name}.sql && -s ${RESTORE_DIR}/${single_db_name}.sql ]]
       then
         create_db_if_not_exist $single_db_name
-        $PSQL -d $single_db_name -f ${RESTORE_DIR}/${single_db_name}.sql 2>dbrestore.log
+        $PSQL -d $single_db_name -f ${RESTORE_DIR}/${single_db_name}.sql 2>>$LOG_FILE
         if [ "$?" -eq 0 ]
         then
           echo "Database Restore Successful."
         else
-          echo "Database Restore Failed."
+          log_error "Database Restore Failed."
         fi
       else
-        echo "Database Dump For $single_db_name is empty or not available."
+        log_error "Database Dump For $single_db_name is empty or not available."
       fi
     else
-      echo "Database file for dump_all not available to restore from"
+      log_error "Database file for dump_all not available to restore from"
     fi
   else
-    echo "Archive does not exist"
+    log_error "Archive does not exist"
   fi
 }
 
@@ -163,18 +168,18 @@ restore_all_dbs() {
     tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null
     if [ -f ${RESTORE_DIR}/postgres.all.sql ]
     then
-      $PSQL postgres -f ${RESTORE_DIR}/postgres.all.sql 2>dbrestore.log
+      $PSQL postgres -f ${RESTORE_DIR}/postgres.all.sql 2>>$LOG_FILE
       if [ "$?" -eq 0 ]
       then
         echo "Database Restore successful."
       else
-        echo "Database Restore failed."
+        log_error "Database Restore failed."
       fi
     else
-      echo "There is no database file available to restore from"
+      log_error "There is no database file available to restore from"
     fi
   else
-    echo "Archive does not exist"
+    log_error "Archive does not exist"
  fi
 }
 
@@ -198,52 +203,48 @@ is_Option() {
 mkdir -p $RESTORE_DIR
 if [ ${#ARGS[@]} -gt 3 ]
 then
-  usage
-  exit
+  usage 0
 elif [ ${#ARGS[@]} -eq 1 ]
 then
-  if [ $(is_Option "$LIST_OPTIONS" ${ARGS[0]}) -eq 1 ]
+  if [ "${ARGS[0]}" == "list_archives" ]
   then
-    ${ARGS[0]}
-    exit
+    list_archives
+  elif [ "${ARGS[0]}" == "help" ]
+  then
+    usage 0
   else
-    usage
-    exit
+    usage 1
   fi
 elif [ ${#ARGS[@]} -eq 2 ]
 then
   if [ "${ARGS[0]}" == "list_databases" ]
   then
     list_databases ${ARGS[1]}
-    exit 0
   else
-    usage
-    exit
+    usage 1
   fi
 elif [ ${#ARGS[@]} -eq 3 ]
 then
   if [ "${ARGS[0]}" != "restore" ]
   then
-    usage
-    exit 1
+    usage 1
   else
     if [ -f ${ARCHIVE_DIR}/${ARGS[1]} ]
     then
       #Get all the databases in that archive
       get_databases ${ARGS[1]}
+
       #check if the requested database is available in the archive
       if [ $(is_Option "$DBS" ${ARGS[2]}) -eq 1 ]
       then
         echo "Restoring Database ${ARGS[2]} And Grants"
         restore_single_db ${ARGS[2]}
-        echo "Tail dbrestore.log for restore log."
-        exit 0
+        echo "Tail ${LOG_FILE} for restore log."
       elif [ "$( echo ${ARGS[2]} | tr '[a-z]' '[A-Z]')" == "ALL" ]
       then
         echo "Restoring All The Database."
         restore_all_dbs
-        echo "Tail dbrestore.log for restore log."
-        exit 0
+        echo "Tail ${LOG_FILE} for restore log."
       else
         echo "There is no database with that name"
       fi
@@ -252,7 +253,7 @@ then
     fi
   fi
 else
-  usage
-  exit
+  usage 1
 fi
 
+exit 0

postgresql/templates/cron-job-backup-postgres.yaml

@@ -41,13 +41,15 @@ spec:
           labels:
 {{ tuple $envAll "postgresql-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 12 }}
         spec:
+          securityContext:
+            fsGroup: 999
           serviceAccountName: {{ $serviceAccountName }}
           restartPolicy: OnFailure
           nodeSelector:
             {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
           containers:
             - name: postgresql-backup
-{{ tuple $envAll "postgresql_backup" | include "helm-toolkit.snippets.image" | indent 14 }}
+{{ tuple $envAll "postgresql" | include "helm-toolkit.snippets.image" | indent 14 }}
 {{ tuple $envAll $envAll.Values.pod.resources.jobs.postgresql_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }}
               command:
                 - /tmp/backup_postgresql.sh

postgresql/values.yaml

@@ -126,7 +126,6 @@ images:
     image_repo_sync: docker.io/docker:17.07.0
     prometheus_postgresql_exporter: docker.io/wrouesnel/postgres_exporter:v0.4.6
     prometheus_postgresql_exporter_create_user: "docker.io/postgres:9.5"
-    postgresql_backup: "docker.io/postgres:9.5"
   pull_policy: "IfNotPresent"
   local_registry:
     active: false