Add postgresql backup capability to postgresql chart

* backup script for postgresql
  * restore script for postgresql
  * cronjob to control backup automation for postgresql
  * add parameters to values.yaml

Change-Id: I5eaa82e824c9f361aa667c431cd93058391f2e60

postgresql/templates/bin/_backup_postgresql.sh.tpl

@@ -0,0 +1,100 @@
+#!/bin/bash
+
+# Copyright 2018 The Openstack-Helm Authors.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+set -x
+export PGPASSFILE=/etc/postgresql/admin_user.conf
+PG_DUMPALL_OPTIONS=$POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS
+BACKUPS_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/current
+ARCHIVE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/archive
+POSTGRESQL_HOST=$(cat /etc/postgresql/admin_user.conf | cut -d: -f 1)
+PG_DUMPALL="pg_dumpall -U $POSTGRESQL_BACKUP_USER -h $POSTGRESQL_HOST"
+
+#Delete files
+delete_files() {
+  files_to_delete=("$@")
+  for f in "${files_to_delete[@]}"
+  do
+    if [ -f $f ]
+    then
+      echo "Deleting file $f."
+      rm -rf $f
+    fi
+  done
+}
+
+#Get the day delta since the archive file backup
+days_difference() {
+  archive_date=$( date --date="$1" +%s )
+  if [ "$?" -ne 0 ]
+  then
+    day_delta=0
+  fi
+  current_date=$( date +%s )
+  date_delta=$(($current_date-$archive_date))
+  if [ "$date_delta" -lt 0 ]
+  then
+    day_delta=0
+  else
+    day_delta=$(($date_delta/86400))
+  fi
+  echo $day_delta
+}
+
+#Create backups directory if it does not exists.
+mkdir -p $BACKUPS_DIR $ARCHIVE_DIR
+
+#Dump all databases
+DATE=$(date +"%Y-%m-%dT%H:%M:%SZ")
+pg_dumpall $POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS -U $POSTGRESQL_BACKUP_USER \
+   -h  $POSTGRESQL_HOST --file=$BACKUPS_DIR/postgres.all.sql 2>dberror.log
+if [[ $? -eq 0 && -s "$BACKUPS_DIR/postgres.all.sql" ]]
+then
+  #Archive the current databases files
+  pushd $BACKUPS_DIR 1>/dev/null
+  tar zcvf $ARCHIVE_DIR/postgres.all.${DATE}.tar.gz *
+  ARCHIVE_RET=$?
+  popd 1>/dev/null
+  #Remove the current backup
+  if [ -d $BACKUPS_DIR ]
+  then
+    rm -rf $BACKUPS_DIR/*.sql
+  fi
+else
+  #TODO: This can be convert into mail alert of alert send to a monitoring system
+  echo "Backup of postgresql failed and need attention."
+  cat dberror.log
+  exit 1
+fi
+
+#Only delete the old archive after a successful archive
+if [ $ARCHIVE_RET -eq 0 ]
+  then
+    if [ "$POSTGRESQL_BACKUP_DAYS_TO_KEEP" -gt 0 ]
+    then
+      echo "Deleting backups older than $POSTGRESQL_BACKUP_DAYS_TO_KEEP days"
+      if [ -d $ARCHIVE_DIR ]
+      then
+        for archive_file in $(ls -1 $ARCHIVE_DIR/*.gz)
+        do
+          archive_date=$( echo $archive_file | awk -F/ '{print $NF}' | cut -d'.' -f 3)
+          if [ "$(days_difference $archive_date)" -gt "$POSTGRESQL_BACKUP_DAYS_TO_KEEP" ]
+          then
+            rm -rf $archive_file
+          fi
+        done
+      fi
+    fi
+fi

postgresql/templates/bin/_restore_postgresql.sh.tpl

@@ -0,0 +1,249 @@
+#!/bin/bash
+
+# Copyright 2018 The Openstack-Helm Authors.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+#set -x
+export PGPASSFILE=/etc/postgresql/admin_user.conf
+ARCHIVE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/archive
+RESTORE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/restore
+POSTGRESQL_HOST=$(cat /etc/postgresql/admin_user.conf | cut -d: -f 1)
+LIST_OPTIONS=(list_archives list_databases)
+ARGS=("$@")
+PSQL="psql -U $POSTGRESQL_BACKUP_USER -h $POSTGRESQL_HOST"
+
+usage() {
+  echo "Usage:"
+  echo "$0 options"
+  echo "============================="
+  echo "options: "
+  echo "list_archives"
+  echo "list_databases archive_filename"
+  echo "restore archive_filename [DB_NAME or ALL/all]"
+}
+
+#Delete file
+delete_files() {
+  files_to_delete=("$@")
+  for f in "${files_to_delete[@]}"
+  do
+    if [ -f $f ]
+    then
+      echo "Deleting file $f."
+      rm -rf $f
+    fi
+  done
+}
+
+#Extract Single Database SQL Dump from pg_dumpall dump file
+extract_single_db_dump() {
+  sed  "/connect.*$2/,\$!d" $1 | sed "/PostgreSQL database dump complete/,\$d" > \
+      ${RESTORE_DIR}/$2.sql
+}
+
+#Display all archives
+list_archives() {
+  if [ -d ${ARCHIVE_DIR} ]
+  then
+    archives=$(find ${ARCHIVE_DIR}/ -iname "*.gz" -print)
+    echo "All Archives"
+    echo "=================================="
+    for archive in $archives
+    do
+      echo $archive | cut -d '/' -f 8
+    done
+    exit 0
+  else
+    echo "Archive directory is not available."
+    exit 1
+  fi
+}
+
+#Return all databases from an archive
+get_databases() {
+  archive_file=$1
+  if [ -e ${ARCHIVE_DIR}/${archive_file} ]
+  then
+    files_to_purge=$(find $RESTORE_DIR/ -iname "*.sql" -print)
+    delete_files $files_to_purge
+    tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null
+    if [ -e ${RESTORE_DIR}/postgres.all.sql ]
+    then
+      DBS=$( grep 'CREATE DATABASE' ${RESTORE_DIR}/postgres.all.sql | awk '{ print $3 }' )
+    else
+      DBS=" "
+    fi
+  else
+    DBS=" "
+  fi
+}
+
+#Display all databases from an archive
+list_databases() {
+  archive_file=$1
+  get_databases $archive_file
+  #echo $DBS
+  if [ -n "$DBS" ]
+  then
+    echo " "
+    echo "Databases in the archive $archive_file"
+    echo "================================================================="
+    for db in $DBS
+    do
+      echo $db
+    done
+  else
+    echo "There is no database in the archive."
+  fi
+
+}
+
+#Restore a single database dump from pg_dumpall dump.
+restore_single_db() {
+  single_db_name=$1
+  if [ -z "$single_db_name" ]
+  then
+    usage
+    exit 1
+  fi
+  if [ -f ${ARCHIVE_DIR}/${archive_file} ]
+  then
+    files_to_purge=$(find $RESTORE_DIR/ -iname "*.sql" -print)
+    delete_files $files_to_purge
+    tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null
+    if [ -f ${RESTORE_DIR}/postgres.all.sql ]
+    then
+      extract_single_db_dump ${RESTORE_DIR}/postgres.all.sql $single_db_name
+      if [[ -f ${RESTORE_DIR}/${single_db_name}.sql && -s ${RESTORE_DIR}/${single_db_name}.sql ]]
+      then
+        $PSQL -d $single_db_name -f ${RESTORE_DIR}/${single_db_name}.sql 2>dbrestore.log
+        if [ "$?" -eq 0 ]
+        then
+          echo "Database Restore Successful."
+        else
+          echo "Database Restore Failed."
+        fi
+      else
+        echo "Database Dump For $single_db_name is empty or not available."
+      fi
+    else
+      echo "Database file for dump_all not available to restore from"
+    fi
+  else
+    echo "Archive does not exist"
+  fi
+}
+
+#Restore all the databases
+restore_all_dbs() {
+  if [ -f ${ARCHIVE_DIR}/${archive_file} ]
+  then
+    files_to_purge=$(find $RESTORE_DIR/ -iname "*.sql" -print)
+    delete_files $files_to_purge
+    tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null
+    if [ -f ${RESTORE_DIR}/postgres.all.sql ]
+    then
+      $PSQL postgres -f ${RESTORE_DIR}/postgres.all.sql 2>dbrestore.log
+      if [ "$?" -eq 0 ]
+      then
+        echo "Database Restore successful."
+      else
+        echo "Database Restore failed."
+      fi
+    else
+      echo "There is no database file available to restore from"
+    fi
+  else
+    echo "Archive does not exist"
+ fi
+}
+
+
+is_Option() {
+  opts=$1
+  param=$2
+  find=0
+  for opt in $opts
+  do
+    if [ "$opt" == "$param" ]
+    then
+      find=1
+    fi
+  done
+  echo $find
+}
+
+#Main
+#Create Restore Directory
+mkdir -p $RESTORE_DIR
+if [ ${#ARGS[@]} -gt 3 ]
+then
+  usage
+  exit
+elif [ ${#ARGS[@]} -eq 1 ]
+then
+  if [ $(is_Option "$LIST_OPTIONS" ${ARGS[0]}) -eq 1 ]
+  then
+    ${ARGS[0]}
+    exit
+  else
+    usage
+    exit
+  fi
+elif [ ${#ARGS[@]} -eq 2 ]
+then
+  if [ "${ARGS[0]}" == "list_databases" ]
+  then
+    list_databases ${ARGS[1]}
+    exit 0
+  else
+    usage
+    exit
+  fi
+elif [ ${#ARGS[@]} -eq 3 ]
+then
+  if [ "${ARGS[0]}" != "restore" ]
+  then
+    usage
+    exit 1
+  else
+    if [ -f ${ARCHIVE_DIR}/${ARGS[1]} ]
+    then
+      #Get all the databases in that archive
+      get_databases ${ARGS[1]}
+      #check if the requested database is available in the archive
+      if [ $(is_Option "$DBS" ${ARGS[2]}) -eq 1 ]
+      then
+        echo "Restoring Database ${ARGS[2]} And Grants"
+        restore_single_db ${ARGS[2]}
+        echo "Tail dbrestore.log for restore log."
+        exit 0
+      elif [ "$( echo ${ARGS[2]} | tr '[a-z]' '[A-Z]')" == "ALL" ]
+      then
+        echo "Restoring All The Database."
+        restore_all_dbs
+        echo "Tail dbrestore.log for restore log."
+        exit 0
+      else
+        echo "There is no database with that name"
+      fi
+    else
+      echo "Archive file not found"
+    fi
+  fi
+else
+  usage
+  exit
+fi
+

postgresql/templates/configmap-bin.yaml

@@ -33,4 +33,8 @@ data:
 {{ tuple "bin/_readiness.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   db_test.sh: |
 {{ tuple "bin/_db_test.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- if .Values.conf.backup.enabled }}
+  backup_postgresql.sh: |
+{{ tuple "bin/_backup_postgresql.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- end }}
 {{- end }}

postgresql/templates/cron-job-backup-postgres.yaml

@@ -0,0 +1,109 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.cron_job_postgresql_backup }}
+{{- $envAll := . }}
+
+{{- $serviceAccountName := "postgresql-backup" }}
+{{ tuple $envAll "postgresql_account" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: postgresql-backup
+  annotations:
+    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+spec:
+  schedule: {{ .Values.jobs.backup_postgresql.cron | quote }}
+  successfulJobsHistoryLimit: {{ .Values.jobs.backup_postgresql.history.success }}
+  failedJobsHistoryLimit: {{ .Values.jobs.backup_postgresql.history.failed }}
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    metadata:
+      labels:
+{{ tuple $envAll "postgresql-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      template:
+        metadata:
+          labels:
+{{ tuple $envAll "postgresql-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 12 }}
+        spec:
+          serviceAccountName: {{ $serviceAccountName }}
+          restartPolicy: OnFailure
+          nodeSelector:
+            {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
+          containers:
+          - command:
+            - /tmp/backup_postgresql.sh
+            env:
+            - name: POSTGRESQL_BACKUP_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  key: POSTGRES_PASSWORD
+                  name: postgresql-admin
+            - name: POSTGRESQL_BACKUP_USER
+              valueFrom:
+                secretKeyRef:
+                  key: POSTGRES_USER
+                  name: postgresql-admin
+            - name: POSTGRESQL_BACKUP_BASE_DIR
+              value: {{ .Values.conf.backup.base_path }}
+            - name: POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS
+              value: {{ .Values.conf.backup.pg_dumpall_options }}
+            - name: POSTGRESQL_BACKUP_DAYS_TO_KEEP
+              value: "{{ .Values.conf.backup.days_of_backup_to_keep }}"
+            - name: POSTGRESQL_POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+{{ tuple $envAll "postgresql_backup" | include "helm-toolkit.snippets.image" | indent 12 }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.postgresql_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 12 }}
+            name: postgresql-backup
+            volumeMounts:
+            - mountPath: /tmp/backup_postgresql.sh
+              name: postgresql-bin
+              readOnly: true
+              subPath: backup_postgresql.sh
+            - mountPath: {{ .Values.conf.backup.base_path }}
+              name: postgresql-backup-dir
+            - name: postgresql-secrets
+              mountPath: /etc/postgresql/admin_user.conf
+              subPath: admin_user.conf
+              readOnly: true
+          restartPolicy: OnFailure
+          securityContext: {}
+          serviceAccount: {{ $serviceAccountName }}
+          serviceAccountName: {{ $serviceAccountName }}
+          volumes:
+          - name: postgresql-secrets
+            secret:
+              secretName: postgresql-secrets
+              defaultMode: 0600
+          - configMap:
+              defaultMode: 365
+              name: postgresql-bin
+            name: postgresql-bin
+          {{- if and .Values.volume.backup.enabled  .Values.manifests.pvc_backup }}
+          - name: postgresql-backup-dir
+            persistentVolumeClaim:
+              claimName: postgresql-backup-data
+          {{- else }}
+          - hostPath:
+              path: {{ .Values.conf.backup.base_path }}
+              type: DirectoryOrCreate
+            name: postgresql-backup-dir
+          {{- end }}
+{{- end }}

postgresql/templates/postgresql-backup-pvc.yaml

@@ -0,0 +1,30 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.volume.backup.enabled .Values.manifests.pvc_backup }}
+---
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: postgresql-backup-data
+spec:
+  accessModes: [ "ReadWriteOnce" ]
+  resources:
+    requests:
+      storage: {{ .Values.volume.backup.size }}
+  storageClassName: {{ .Values.volume.backup.class_name }}
+{{- end }}
+

postgresql/templates/secrets-etc.yaml

@@ -0,0 +1,28 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.secret_etc }}
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: postgresql-secrets
+type: Opaque
+data:
+  admin_user.conf: {{ tuple "secrets/_admin_user.conf.tpl" . | include "helm-toolkit.utils.template"  | b64enc }}
+{{- end }}
+

postgresql/templates/secrets/_admin_user.conf.tpl

@@ -0,0 +1,17 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{ tuple "postgresql" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}:*:*:{{ .Values.endpoints.postgresql.auth.admin.username }}:{{ .Values.endpoints.postgresql.auth.admin.password }}

postgresql/values.yaml

@@ -86,6 +86,13 @@ pod:
         requests:
           memory: "128Mi"
           cpu: "100m"
+      postgresql_backup:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
 
 # using dockerhub postgresql: https://hub.docker.com/r/library/postgres/tags/
 images:
@@ -95,6 +102,7 @@ images:
     image_repo_sync: docker.io/docker:17.07.0
     prometheus_postgresql_exporter: docker.io/wrouesnel/postgres_exporter:v0.4.6
     prometheus_postgresql_exporter_create_user: "docker.io/postgres:9.5"
+    postgresql_backup: "docker.io/postgres:9.5"
   pull_policy: "IfNotPresent"
   local_registry:
     active: false
@@ -124,6 +132,9 @@ labels:
   prometheus_postgresql_exporter:
     node_selector_key: openstack-control-plane
     node_selector_value: enabled
+  job:
+    node_selector_key: openstack-control-plane
+    node_selector_value: enabled
 
 dependencies:
   dynamic:
@@ -155,6 +166,10 @@ dependencies:
           service: postgresql
       jobs:
         - prometheus-postgresql-exporter-create-user
+    postgresql-backup:
+      services:
+        - endpoint: internal
+          service: postgresql
 
 monitoring:
   prometheus:
@@ -162,11 +177,29 @@ monitoring:
     postgresql_exporter:
       scrape: true
 
+volume:
+  backup:
+    enabled: true
+    class_name: general
+    size: 5Gi
+
+jobs:
+  backup_postgresql:
+    cron: "0 0 * * *"
+    history:
+      success: 3
+      failed: 1
+
 conf:
   debug: false
   postgresql:
     max_connections: 100
     shared_buffers: 128MB
+  backup:
+    enabled: true
+    base_path: /var/backup
+    days_of_backup_to_keep: 3
+    pg_dumpall_options: null
 
 secrets:
   postgresql:
@@ -222,8 +255,11 @@ manifests:
   configmap_bin: true
   job_image_repo_sync: true
   secret_admin: true
+  secret_etc: true
   service: true
   statefulset: true
+  cron_job_postgresql_backup: false
+  pvc_backup: false
   monitoring:
     prometheus:
       configmap_bin: true