Backup/restore enhancements
This patchset introduces the framework by which all OSH-based database systems can use to backup and restore their databases. The framework is refactored from the Postgresql backup and restore logic. This will prevent alot of code duplication in the backup restore scripts across each cluster. In the process, some improvements needed to be made: 1) Removing the need for 2 separate containers to do the backup and restore work to a remote gateway. This simplifies the design and enables a higher level of robustness. 2) Adding separate "days to keep" config value for remote backup files, as there may be different requirements for the remote files than the local backup files. 3) Adding capability to send Storage_Policy when creating the remote RGW swift container. 4) Making coding style improvement for readability and maintainability. 5) Fixing a deployment bug that occurs when remote backup is disabled. Change-Id: I3a3482ad67320e89f04305b17da79abf7ad6eb45
This commit is contained in:
Parsons, Cliff (cp769u)
parent
e53d28718d
commit
5a2babd514
367
helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
Executable file
367
helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
Executable file
@ -0,0 +1,367 @@
|
||||
{{- define "helm-toolkit.scripts.db-backup-restore.backup_main" }}
|
||||
#!/bin/bash
|
||||
|
||||
# This file contains a database backup framework which database scripts
|
||||
# can use to perform a backup. The idea here is that the database-specific
|
||||
# functions will be implemented by the various databases using this script
|
||||
# (like mariadb, postgresql or etcd for example). The database-specific
|
||||
# script will need to first "source" this file like this:
|
||||
# source /tmp/backup_main.sh
|
||||
#
|
||||
# Then the script should call the main backup function (backup_databases):
|
||||
# backup_databases
|
||||
#
|
||||
# No arguments required. However, the framework will require the
|
||||
# following variables to be exported:
|
||||
#
|
||||
# export DB_NAMESPACE Namespace where the database(s) reside
|
||||
# export DB_NAME Name of the database system
|
||||
# export LOCAL_DAYS_TO_KEEP Number of days to keep the local backups
|
||||
# export REMOTE_DAYS_TO_KEEP Number of days to keep the remote backups
|
||||
# export ARCHIVE_DIR Local location where the backup tarballs should
|
||||
# be stored. (full directory path)
|
||||
# export REMOTE_BACKUP_ENABLED "true" if remote backup enabled; false
|
||||
# otherwise
|
||||
# export CONTAINER_NAME Name of the container on the RGW to store
|
||||
# the backup tarball.
|
||||
# export STORAGE_POLICY Name of the storage policy defined on the
|
||||
# RGW which is intended to store backups.
|
||||
# RGW access variables:
|
||||
# export OS_REGION_NAME Name of the region the RGW resides in
|
||||
# export OS_AUTH_URL Keystone URL associated with the RGW
|
||||
# export OS_PROJECT_NAME Name of the project associated with the
|
||||
# keystone user
|
||||
# export OS_USERNAME Name of the keystone user
|
||||
# export OS_PASSWORD Password of the keystone user
|
||||
# export OS_USER_DOMAIN_NAME Keystone domain the project belongs to
|
||||
# export OS_PROJECT_DOMAIN_NAME Keystone domain the user belongs to
|
||||
# export OS_IDENTITY_API_VERSION Keystone API version to use
|
||||
#
|
||||
# The following variables are optional:
|
||||
# export RGW_TIMEOUT Number of seconds to wait for the
|
||||
# connection to the RGW to be available
|
||||
# when sending a backup to the RGW. Default
|
||||
# is 1800 (30 minutes).
|
||||
#
|
||||
# The database-specific functions that need to be implemented are:
|
||||
# dump_databases_to_directory <directory> <err_logfile>
|
||||
# where:
|
||||
# <directory> is the full directory path to dump the database files
|
||||
# into. This is a temporary directory for this backup only.
|
||||
# <err_logfile> is the full directory path where error logs are to be
|
||||
# written by the application.
|
||||
# returns: 0 if no errors; 1 if any errors occurred
|
||||
#
|
||||
# This function is expected to dump the database file(s) to the specified
|
||||
# directory path. If this function completes successfully (returns 0), the
|
||||
# framework will automatically tar/zip the files in that directory and
|
||||
# name the tarball appropriately according to the proper conventions.
|
||||
#
|
||||
# The functions in this file will take care of:
|
||||
# 1) Calling "dump_databases_to_directory" and then compressing the files,
|
||||
# naming the tarball properly, and then storing it locally at the specified
|
||||
# local directory.
|
||||
# 2) Sending the tarball built to the remote gateway, to be stored in the
|
||||
# container configured to store database backups.
|
||||
# 3) Removing local backup tarballs which are older than the number of days
|
||||
# specified by the "LOCAL_DAYS_TO_KEEP" variable.
|
||||
# 4) Removing remote backup tarballs (from the remote gateway) which are older
|
||||
# than the number of days specified by the "REMOTE_DAYS_TO_KEEP" variable.
|
||||
#
|
||||
|
||||
# Note: not using set -e in this script because more elaborate error handling
|
||||
# is needed.
|
||||
set -x
|
||||
|
||||
log_backup_error_exit() {
|
||||
MSG=$1
|
||||
ERRCODE=$2
|
||||
log ERROR "${DB_NAME}_backup" "${MSG}"
|
||||
rm -f $ERR_LOG_FILE
|
||||
rm -rf $TMP_DIR
|
||||
exit $ERRCODE
|
||||
}
|
||||
|
||||
log() {
|
||||
#Log message to a file or stdout
|
||||
#TODO: This can be convert into mail alert of alert send to a monitoring system
|
||||
#Params: $1 log level
|
||||
#Params: $2 service
|
||||
#Params: $3 message
|
||||
#Params: $4 Destination
|
||||
LEVEL=$1
|
||||
SERVICE=$2
|
||||
MSG=$3
|
||||
DEST=$4
|
||||
DATE=$(date +"%m-%d-%y %H:%M:%S")
|
||||
if [[ -z "$DEST" ]]; then
|
||||
echo "${DATE} ${LEVEL}: $(hostname) ${SERVICE}: ${MSG}"
|
||||
else
|
||||
echo "${DATE} ${LEVEL}: $(hostname) ${SERVICE}: ${MSG}" >>$DEST
|
||||
fi
|
||||
}
|
||||
|
||||
#Get the day delta since the archive file backup
|
||||
seconds_difference() {
|
||||
ARCHIVE_DATE=$( date --date="$1" +%s )
|
||||
if [[ $? -ne 0 ]]; then
|
||||
SECOND_DELTA=0
|
||||
fi
|
||||
CURRENT_DATE=$( date +%s )
|
||||
SECOND_DELTA=$(($CURRENT_DATE-$ARCHIVE_DATE))
|
||||
if [[ "$SECOND_DELTA" -lt 0 ]]; then
|
||||
SECOND_DELTA=0
|
||||
fi
|
||||
echo $SECOND_DELTA
|
||||
}
|
||||
|
||||
# Send the specified tarball file at the specified filepath to the
|
||||
# remote gateway.
|
||||
send_to_remote_server() {
|
||||
FILEPATH=$1
|
||||
FILE=$2
|
||||
|
||||
# Grab the list of containers on the remote site
|
||||
RESULT=$(openstack container list 2>&1)
|
||||
|
||||
if [[ $? -eq 0 ]]; then
|
||||
echo $RESULT | grep $CONTAINER_NAME
|
||||
if [[ $? -ne 0 ]]; then
|
||||
# Find the swift URL from the keystone endpoint list
|
||||
SWIFT_URL=$(openstack catalog list -f value | grep -A5 swift | grep public | awk '{print $2}')
|
||||
|
||||
# Get a token from keystone
|
||||
TOKEN=$(openstack token issue -f value -c id)
|
||||
|
||||
# Create the container
|
||||
RES_FILE=$(mktemp -p /tmp)
|
||||
curl -g -i -X PUT ${SWIFT_URL}/${CONTAINER_NAME} \
|
||||
-H "X-Auth-Token: ${TOKEN}" \
|
||||
-H "X-Storage-Policy: ${STORAGE_POLICY}" 2>&1 > $RES_FILE
|
||||
|
||||
if [[ $? -ne 0 || $(grep "HTTP" $RES_FILE | awk '{print $2}') -ge 400 ]]; then
|
||||
log ERROR "${DB_NAME}_backup" "Error creating container ${CONTAINER_NAME}"
|
||||
cat $RES_FILE
|
||||
rm -f $RES_FILE
|
||||
return 1
|
||||
fi
|
||||
rm -f $RES_FILE
|
||||
|
||||
swift stat $CONTAINER_NAME
|
||||
if [[ $? -ne 0 ]]; then
|
||||
log ERROR "${DB_NAME}_backup" "Error retrieving container ${CONTAINER_NAME} details after creation."
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo $RESULT | grep "HTTP 401"
|
||||
if [[ $? -eq 0 ]]; then
|
||||
log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}"
|
||||
return 1
|
||||
else
|
||||
echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions"
|
||||
if [[ $? -eq 0 ]]; then
|
||||
log ERROR "${DB_NAME}_backup" "Could not reach the RGW: ${RESULT}"
|
||||
# In this case, keystone or the site/node may be temporarily down.
|
||||
# Return slightly different error code so the calling code can retry
|
||||
return 2
|
||||
else
|
||||
log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Create an object to store the file
|
||||
openstack object create --name $FILE $CONTAINER_NAME $FILEPATH/$FILE || log ERROR "${DB_NAME}_backup" "Cannot create container object ${FILE}!"
|
||||
openstack object show $CONTAINER_NAME $FILE
|
||||
if [[ $? -ne 0 ]]; then
|
||||
log ERROR "${DB_NAME}_backup" "Error retrieving container object $FILE after creation."
|
||||
return 1
|
||||
fi
|
||||
|
||||
log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully."
|
||||
return 0
|
||||
}
|
||||
|
||||
# This function attempts to store the built tarball to the remote gateway,
|
||||
# with built-in logic to handle error cases like:
|
||||
# 1) Network connectivity issues - retries for a specific amount of time
|
||||
# 2) Authorization errors - immediately logs an ERROR and exits
|
||||
store_backup_remotely() {
|
||||
FILEPATH=$1
|
||||
FILE=$2
|
||||
|
||||
# If the RGW_TIMEOUT has already been set, use that value, otherwise give it
|
||||
# a default value.
|
||||
if [[ -z $RGW_TIMEOUT ]]; then
|
||||
RGW_TIMEOUT=1800
|
||||
fi
|
||||
|
||||
ERROR_SEEN=false
|
||||
DONE=false
|
||||
TIMEOUT_EXP=$(( $(date +%s) + $RGW_TIMEOUT ))
|
||||
while [[ $DONE == "false" ]]; do
|
||||
# Store the new archive to the remote backup storage facility.
|
||||
send_to_remote_server $FILEPATH $FILE
|
||||
|
||||
# Check if successful
|
||||
if [[ $? -eq 0 ]]; then
|
||||
log INFO "${DB_NAME}_backup" "Backup file ${FILE} successfully sent to RGW."
|
||||
DONE=true
|
||||
elif [[ $? -eq 2 ]]; then
|
||||
# Temporary failure occurred. We need to retry if we have not timed out
|
||||
log WARN "${DB_NAME}_backup" "Backup file ${FILE} could not be sent to RGW due to connection issue."
|
||||
DELTA=$(( TIMEOUT_EXP - $(date +%s) ))
|
||||
if [[ $DELTA -lt 0 ]]; then
|
||||
DONE=true
|
||||
log ERROR "${DB_NAME}_backup" "Timed out waiting for RGW to become available."
|
||||
ERROR_SEEN=true
|
||||
else
|
||||
log INFO "${DB_NAME}_backup" "Sleeping 30 seconds waiting for RGW to become available..."
|
||||
sleep 30
|
||||
log INFO "${DB_NAME}_backup" "Retrying..."
|
||||
fi
|
||||
else
|
||||
log ERROR "${DB_NAME}_backup" "Backup file ${FILE} could not be sent to the RGW."
|
||||
ERROR_SEEN=true
|
||||
DONE=true
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ $ERROR_SEEN == "true" ]]; then
|
||||
log ERROR "${DB_NAME}_backup" "Errors encountered. Exiting."
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
remove_old_local_archives() {
|
||||
log INFO "${DB_NAME}_backup" "Deleting backups older than ${LOCAL_DAYS_TO_KEEP} days"
|
||||
if [[ -d $ARCHIVE_DIR ]]; then
|
||||
for ARCHIVE_FILE in $(ls -1 $ARCHIVE_DIR/*.gz); do
|
||||
ARCHIVE_DATE=$( echo $ARCHIVE_FILE | awk -F/ '{print $NF}' | cut -d'.' -f 4)
|
||||
if [[ "$(seconds_difference $ARCHIVE_DATE)" -gt "$(($LOCAL_DAYS_TO_KEEP*86400))" ]]; then
|
||||
log INFO "${DB_NAME}_backup" "Deleting file $ARCHIVE_FILE."
|
||||
rm -rf $ARCHIVE_FILE
|
||||
if [[ $? -ne 0 ]]; then
|
||||
# Log error but don't exit so we can finish the script
|
||||
# because at this point we haven't sent backup to RGW yet
|
||||
log ERROR "${DB_NAME}_backup" "Cannot remove ${ARCHIVE_FILE}"
|
||||
fi
|
||||
else
|
||||
log INFO "${DB_NAME}_backup" "Keeping file ${ARCHIVE_FILE}."
|
||||
fi
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
remove_old_remote_archives() {
|
||||
log INFO "${DB_NAME}_backup" "Deleting backups older than ${REMOTE_DAYS_TO_KEEP} days"
|
||||
BACKUP_FILES=$(mktemp -p /tmp)
|
||||
DB_BACKUP_FILES=$(mktemp -p /tmp)
|
||||
|
||||
openstack object list $CONTAINER_NAME > $BACKUP_FILES
|
||||
if [[ $? -ne 0 ]]; then
|
||||
log_backup_error_exit "Could not obtain a list of current backup files in the RGW" 1
|
||||
fi
|
||||
|
||||
# Filter out other types of backup files
|
||||
cat $BACKUP_FILES | grep $DB_NAME | grep $DB_NAMESPACE | awk '{print $2}' > $DB_BACKUP_FILES
|
||||
|
||||
for ARCHIVE_FILE in $(cat $DB_BACKUP_FILES); do
|
||||
ARCHIVE_DATE=$( echo $ARCHIVE_FILE | awk -F/ '{print $NF}' | cut -d'.' -f 4)
|
||||
if [[ "$(seconds_difference ${ARCHIVE_DATE})" -gt "$((${REMOTE_DAYS_TO_KEEP}*86400))" ]]; then
|
||||
log INFO "${DB_NAME}_backup" "Deleting file ${ARCHIVE_FILE} from the RGW"
|
||||
openstack object delete $CONTAINER_NAME $ARCHIVE_FILE || log_backup_error_exit "Cannot delete container object ${ARCHIVE_FILE}!" 1
|
||||
fi
|
||||
done
|
||||
|
||||
# Cleanup now that we're done.
|
||||
rm -f $BACKUP_FILES $DB_BACKUP_FILES
|
||||
}
|
||||
|
||||
# Main function to backup the databases. Calling functions need to supply:
|
||||
# 1) The directory where the final backup will be kept after it is compressed.
|
||||
# 2) A temporary directory to use for placing database files to be compressed.
|
||||
# Note: this temp directory will be deleted after backup is done.
|
||||
backup_databases() {
|
||||
# Create necessary directories if they do not exist.
|
||||
mkdir -p $ARCHIVE_DIR || log_backup_error_exit "Cannot create directory ${ARCHIVE_DIR}!"
|
||||
export TMP_DIR=$(mktemp -d) || log_backup_error_exit "Cannot create temp directory!"
|
||||
|
||||
# Create temporary log file
|
||||
export ERR_LOG_FILE=$(mktemp -p /tmp) || log_backup_error_exit "Cannot create log file!"
|
||||
|
||||
# It is expected that this function will dump the database files to the $TMP_DIR
|
||||
dump_databases_to_directory $TMP_DIR $ERR_LOG_FILE
|
||||
|
||||
# If successful, there should be at least one file in the TMP_DIR
|
||||
if [[ $? -ne 0 || $(ls $TMP_DIR | wc -w) -eq 0 ]]; then
|
||||
cat $ERR_LOG_FILE
|
||||
log_backup_error_exit "Backup of the ${DB_NAME} database failed and needs attention."
|
||||
fi
|
||||
|
||||
log INFO "${DB_NAME}_backup" "Databases dumped successfully. Creating tarball..."
|
||||
|
||||
NOW=$(date +"%Y-%m-%dT%H:%M:%SZ")
|
||||
TARBALL_FILE="${DB_NAME}.${DB_NAMESPACE}.all.${NOW}.tar.gz"
|
||||
|
||||
cd $TMP_DIR || log_backup_error_exit "Cannot change to directory $TMP_DIR"
|
||||
|
||||
#Archive the current database files
|
||||
tar zcvf $ARCHIVE_DIR/$TARBALL_FILE *
|
||||
if [[ $? -ne 0 ]]; then
|
||||
log_backup_error_exit "Backup tarball could not be created."
|
||||
fi
|
||||
|
||||
# Get the size of the file
|
||||
ARCHIVE_SIZE=$(ls -l $ARCHIVE_DIR/$TARBALL_FILE | awk '{print $5}')
|
||||
|
||||
log INFO "${DB_NAME}_backup" "Tarball $TARBALL_FILE created successfully."
|
||||
|
||||
cd $ARCHIVE_DIR
|
||||
|
||||
# Remove the temporary directory and files as they are no longer needed.
|
||||
rm -rf $TMP_DIR
|
||||
rm -f $ERR_LOG_FILE
|
||||
|
||||
#Only delete the old archive after a successful archive
|
||||
if [[ "$LOCAL_DAYS_TO_KEEP" -gt 0 ]]; then
|
||||
remove_old_local_archives
|
||||
fi
|
||||
|
||||
if $REMOTE_BACKUP_ENABLED; then
|
||||
store_backup_remotely $ARCHIVE_DIR $TARBALL_FILE
|
||||
if [[ $? -ne 0 ]]; then
|
||||
log_backup_error_exit "Backup could not be sent to remote RGW."
|
||||
fi
|
||||
|
||||
#Only delete the old archive after a successful archive
|
||||
if [[ "$REMOTE_DAYS_TO_KEEP" -gt 0 ]]; then
|
||||
remove_old_remote_archives
|
||||
fi
|
||||
|
||||
# Turn off trace just for a clearer printout of backup status - for manual backups, mainly.
|
||||
set +x
|
||||
echo "=================================================================="
|
||||
echo "Local backup and backup to remote RGW successful!"
|
||||
echo "Backup archive name: $TARBALL_FILE"
|
||||
echo "Backup archive size: $ARCHIVE_SIZE"
|
||||
echo "=================================================================="
|
||||
set -x
|
||||
else
|
||||
# Remote backup is not enabled. This is ok; at least we have a local backup.
|
||||
log INFO "${DB_NAME}_backup" "Skipping remote backup, as it is not enabled."
|
||||
|
||||
# Turn off trace just for a clearer printout of backup status - for manual backups, mainly.
|
||||
set +x
|
||||
echo "=================================================================="
|
||||
echo "Local backup successful!"
|
||||
echo "Backup archive name: $TARBALL_FILE"
|
||||
echo "Backup archive size: $ARCHIVE_SIZE"
|
||||
echo "=================================================================="
|
||||
set -x
|
||||
fi
|
||||
}
|
||||
{{- end }}
|
||||
375
helm-toolkit/templates/scripts/db-backup-restore/_restore_main.sh.tpl
Executable file
375
helm-toolkit/templates/scripts/db-backup-restore/_restore_main.sh.tpl
Executable file
@ -0,0 +1,375 @@
|
||||
{{- define "helm-toolkit.scripts.db-backup-restore.restore_main" }}
|
||||
#!/bin/bash
|
||||
|
||||
# This file contains a database restore framework which database scripts
|
||||
# can use to perform a backup. The idea here is that the database-specific
|
||||
# functions will be implemented by the various databases using this script
|
||||
# (like mariadb, postgresql or etcd for example). The database-specific
|
||||
# script will need to first "source" this file like this:
|
||||
# source /tmp/restore_main.sh
|
||||
#
|
||||
# Then the script should call the main CLI function (cli_main):
|
||||
# cli_main <arg_list>
|
||||
# where:
|
||||
# <arg_list> is the list of arguments given by the user
|
||||
#
|
||||
# The framework will require the following variables to be exported:
|
||||
#
|
||||
# export DB_NAMESPACE Namespace where the database(s) reside
|
||||
# export DB_NAME Name of the database system
|
||||
# export ARCHIVE_DIR Location where the backup tarballs should
|
||||
# be stored. (full directory path which
|
||||
# should already exist)
|
||||
# export CONTAINER_NAME Name of the container on the RGW where
|
||||
# the backups are stored.
|
||||
# RGW access variables:
|
||||
# export OS_REGION_NAME Name of the region the RGW resides in
|
||||
# export OS_AUTH_URL Keystone URL associated with the RGW
|
||||
# export OS_PROJECT_NAME Name of the project associated with the
|
||||
# keystone user
|
||||
# export OS_USERNAME Name of the keystone user
|
||||
# export OS_PASSWORD Password of the keystone user
|
||||
# export OS_USER_DOMAIN_NAME Keystone domain the project belongs to
|
||||
# export OS_PROJECT_DOMAIN_NAME Keystone domain the user belongs to
|
||||
# export OS_IDENTITY_API_VERSION Keystone API version to use
|
||||
#
|
||||
# The database-specific functions that need to be implemented are:
|
||||
# get_databases
|
||||
# where:
|
||||
# <tmp_dir> is the full directory path where the decompressed
|
||||
# database files reside
|
||||
# <db_file> is the full path of the file to write the database
|
||||
# names into, one database per line
|
||||
# returns: 0 if no errors; 1 if any errors occurred
|
||||
#
|
||||
# This function is expected to extract the database names from the
|
||||
# uncompressed database files found in the given "tmp_dir", which is
|
||||
# the staging directory for database restore. The database names
|
||||
# should be written to the given "db_file", one database name per
|
||||
# line.
|
||||
#
|
||||
# restore_single_db
|
||||
# where:
|
||||
# <db_name> is the name of the database to be restored
|
||||
# <tmp_dir> is the full directory path where the decompressed
|
||||
# database files reside
|
||||
# returns: 0 if no errors; 1 if any errors occurred
|
||||
#
|
||||
# This function is expected to restore the database given as "db_name"
|
||||
# using the database files located in the "tmp_dir". The framework
|
||||
# will delete the "tmp_dir" and the files in it after the restore is
|
||||
# complete.
|
||||
#
|
||||
# restore_all_dbs
|
||||
# where:
|
||||
# <tmp_dir> is the full directory path where the decompressed
|
||||
# database files reside
|
||||
# returns: 0 if no errors; 1 if any errors occurred
|
||||
#
|
||||
# This function is expected to restore all of the databases which
|
||||
# are backed up in the database files located in the "tmp_dir". The
|
||||
# framework will delete the "tmp_dir" and the files in it after the
|
||||
# restore is complete.
|
||||
#
|
||||
# The functions in this file will take care of:
|
||||
# 1) The CLI parameter parsing for the arguments passed in by the user.
|
||||
# 2) The listing of either local or remote archive files at the request
|
||||
# of the user.
|
||||
# 3) The retrieval/download of an archive file located either in the local
|
||||
# file system or remotely stored on an RGW.
|
||||
# 4) Calling either "restore_single_db" or "restore_all_dbs" when the user
|
||||
# chooses to restore a database or all databases.
|
||||
# 5) The framework will call "get_databases" when it needs a list of
|
||||
# databases when the user requests a database list or when the user
|
||||
# requests to restore a single database (to ensure it exists in the
|
||||
# archive).
|
||||
#
|
||||
|
||||
export LOG_FILE=/tmp/dbrestore.log
|
||||
|
||||
usage() {
|
||||
ret_val=$1
|
||||
echo "Usage:"
|
||||
echo "Restore command options"
|
||||
echo "============================="
|
||||
echo "help"
|
||||
echo "list_archives [remote]"
|
||||
echo "list_databases <archive_filename> [remote]"
|
||||
echo "restore <archive_filename> <db_specifier> [remote]"
|
||||
echo " where <db_specifier> = <dbname> | ALL"
|
||||
clean_and_exit $ret_val ""
|
||||
}
|
||||
|
||||
#Exit cleanly with some message and return code
|
||||
clean_and_exit() {
|
||||
RETCODE=$1
|
||||
MSG=$2
|
||||
|
||||
# Clean/remove temporary directories/files
|
||||
rm -rf $TMP_DIR
|
||||
rm -f $DB_FILE
|
||||
|
||||
if [[ "x${MSG}" != "x" ]]; then
|
||||
echo $MSG
|
||||
fi
|
||||
exit $RETCODE
|
||||
}
|
||||
|
||||
# Retrieve a list of archives from the RGW.
|
||||
retrieve_remote_listing() {
|
||||
RESULT=$(openstack container show $CONTAINER_NAME 2>&1)
|
||||
if [[ $? -eq 0 ]]; then
|
||||
# Get the list, ensureing that we only pick up the right kind of backups from the
|
||||
# requested namespace
|
||||
openstack object list $CONTAINER_NAME | grep $DB_NAME | grep $DB_NAMESPACE | awk '{print $2}' > $TMP_DIR/archive_list
|
||||
if [[ $? -ne 0 ]]; then
|
||||
echo "Container object listing could not be obtained."
|
||||
return 1
|
||||
else
|
||||
echo "Archive listing successfully retrieved."
|
||||
fi
|
||||
else
|
||||
echo $RESULT | grep "HTTP 401"
|
||||
if [[ $? -eq 0 ]]; then
|
||||
echo "Could not access the container: ${RESULT}"
|
||||
return 1
|
||||
else
|
||||
echo $RESULT | grep "ConnectionError"
|
||||
if [[ $? -eq 0 ]]; then
|
||||
echo "Could not reach the RGW: ${RESULT}"
|
||||
# In this case, keystone or the site/node may be temporarily down.
|
||||
# Return slightly different error code so the calling code can retry
|
||||
return 2
|
||||
else
|
||||
echo "Container $CONTAINER_NAME does not exist: ${RESULT}"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# Retrieve a single archive from the RGW.
|
||||
retrieve_remote_archive() {
|
||||
ARCHIVE=$1
|
||||
|
||||
RESULT=$(openstack object save --file $TMP_DIR/$ARCHIVE $CONTAINER_NAME $ARCHIVE 2>&1)
|
||||
if [[ $? -ne 0 ]]; then
|
||||
echo $RESULT | grep "HTTP 401"
|
||||
if [[ $? -eq 0 ]]; then
|
||||
echo "Could not access the archive: ${RESULT}"
|
||||
return 1
|
||||
else
|
||||
echo $RESULT | grep "ConnectionError"
|
||||
if [[ $? -eq 0 ]]; then
|
||||
echo "Could not reach the RGW: ${RESULT}"
|
||||
# In this case, keystone or the site/node may be temporarily down.
|
||||
# Return slightly different error code so the calling code can retry
|
||||
return 2
|
||||
else
|
||||
echo "Archive ${ARCHIVE} could not be retrieved: ${RESULT}"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo "Archive $ARCHIVE successfully retrieved."
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# Display all archives
|
||||
list_archives() {
|
||||
REMOTE=$1
|
||||
|
||||
if [[ "x${REMOTE^^}" == "xREMOTE" ]]; then
|
||||
retrieve_remote_listing
|
||||
if [[ $? -eq 0 && -e $TMP_DIR/archive_list ]]; then
|
||||
echo
|
||||
echo "All Archives from RGW Data Store"
|
||||
echo "=============================================="
|
||||
cat $TMP_DIR/archive_list
|
||||
clean_and_exit 0 ""
|
||||
else
|
||||
clean_and_exit 1 "ERROR: Archives could not be retrieved from the RGW."
|
||||
fi
|
||||
elif [[ "x${REMOTE}" == "x" ]]; then
|
||||
if [[ -d $ARCHIVE_DIR ]]; then
|
||||
archives=$(find $ARCHIVE_DIR/ -iname "*.gz" -print)
|
||||
echo
|
||||
echo "All Local Archives"
|
||||
echo "=============================================="
|
||||
for archive in $archives
|
||||
do
|
||||
echo $archive | cut -d '/' -f 8
|
||||
done
|
||||
clean_and_exit 0 ""
|
||||
else
|
||||
clean_and_exit 1 "ERROR: Local archive directory is not available."
|
||||
fi
|
||||
else
|
||||
usage 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Retrieve the archive from the desired location and decompress it into
|
||||
# the restore directory
|
||||
get_archive() {
|
||||
ARCHIVE_FILE=$1
|
||||
REMOTE=$2
|
||||
|
||||
if [[ "x$REMOTE" == "xremote" ]]; then
|
||||
retrieve_remote_archive $ARCHIVE_FILE
|
||||
if [[ $? -ne 0 ]]; then
|
||||
clean_and_exit 1 "ERROR: Could not retrieve remote archive: $ARCHIVE_FILE"
|
||||
fi
|
||||
elif [[ "x$REMOTE" == "x" ]]; then
|
||||
if [[ -e $ARCHIVE_DIR/$ARCHIVE_FILE ]]; then
|
||||
cp $ARCHIVE_DIR/$ARCHIVE_FILE $TMP_DIR/$ARCHIVE_FILE
|
||||
if [[ $? -ne 0 ]]; then
|
||||
clean_and_exit 1 "ERROR: Could not copy local archive to restore directory."
|
||||
fi
|
||||
else
|
||||
clean_and_exit 1 "ERROR: Local archive file could not be found."
|
||||
fi
|
||||
else
|
||||
usage 1
|
||||
fi
|
||||
|
||||
echo "Decompressing archive $ARCHIVE_FILE..."
|
||||
cd $TMP_DIR
|
||||
tar zxvf - < $TMP_DIR/$ARCHIVE_FILE 1>/dev/null
|
||||
if [[ $? -ne 0 ]]; then
|
||||
clean_and_exit 1 "ERROR: Archive decompression failed."
|
||||
fi
|
||||
}
|
||||
|
||||
# Display all databases from an archive
|
||||
list_databases() {
|
||||
ARCHIVE_FILE=$1
|
||||
REMOTE=$2
|
||||
WHERE="local"
|
||||
|
||||
if [[ "x${REMOTE}" != "x" ]]; then
|
||||
WHERE="remote"
|
||||
fi
|
||||
|
||||
# Get the archive from the source location (local/remote)
|
||||
get_archive $ARCHIVE_FILE $REMOTE
|
||||
|
||||
# Expectation is that the database listing will be put into
|
||||
# the given file one database per line
|
||||
get_databases $TMP_DIR $DB_FILE
|
||||
if [[ "$?" -ne 0 ]]; then
|
||||
clean_and_exit 1 "ERROR: Could not list databases."
|
||||
fi
|
||||
|
||||
if [[ -f "$DB_FILE" ]]; then
|
||||
echo " "
|
||||
echo "Databases in the $WHERE archive $ARCHIVE_FILE"
|
||||
echo "================================================================================"
|
||||
cat $DB_FILE
|
||||
else
|
||||
echo "There is no database in the archive."
|
||||
fi
|
||||
}
|
||||
|
||||
# Return 1 if the given database exists in the database file. 0 otherwise.
|
||||
database_exists() {
|
||||
DB=$1
|
||||
|
||||
grep "${DB}" ${DB_FILE}
|
||||
if [[ $? -eq 0 ]]; then
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# This is the main CLI interpreter function
|
||||
cli_main() {
|
||||
ARGS=("$@")
|
||||
|
||||
# Create temp directory for a staging area to decompress files into
|
||||
export TMP_DIR=$(mktemp -d)
|
||||
|
||||
# Create a temp file for storing list of databases (if needed)
|
||||
export DB_FILE=$(mktemp -p /tmp)
|
||||
|
||||
if [[ ${#ARGS[@]} -gt 4 ]]; then
|
||||
usage 1
|
||||
elif [[ ${#ARGS[@]} -eq 1 ]]; then
|
||||
if [[ "${ARGS[0]}" == "list_archives" ]]; then
|
||||
list_archives
|
||||
clean_and_exit 0 ""
|
||||
elif [[ "${ARGS[0]}" == "help" ]]; then
|
||||
usage 0
|
||||
else
|
||||
usage 1
|
||||
fi
|
||||
elif [[ ${#ARGS[@]} -eq 2 ]]; then
|
||||
if [[ "${ARGS[0]}" == "list_databases" ]]; then
|
||||
list_databases ${ARGS[1]}
|
||||
clean_and_exit 0 ""
|
||||
elif [[ "${ARGS[0]}" == "list_archives" ]]; then
|
||||
list_archives ${ARGS[1]}
|
||||
clean_and_exit 0 ""
|
||||
else
|
||||
usage 1
|
||||
fi
|
||||
elif [[ ${#ARGS[@]} -eq 3 || ${#ARGS[@]} -eq 4 ]]; then
|
||||
if [[ "${ARGS[0]}" == "list_databases" ]]; then
|
||||
list_databases ${ARGS[1]} ${ARGS[2]}
|
||||
clean_and_exit 0 ""
|
||||
elif [[ "${ARGS[0]}" != "restore" ]]; then
|
||||
usage 1
|
||||
else
|
||||
ARCHIVE=${ARGS[1]}
|
||||
DB_SPEC=${ARGS[2]}
|
||||
REMOTE=""
|
||||
if [[ ${#ARGS[@]} -eq 4 ]]; then
|
||||
REMOTE=${ARGS[3]}
|
||||
fi
|
||||
|
||||
#Get all the databases in that archive
|
||||
get_archive $ARCHIVE $REMOTE
|
||||
|
||||
if [[ "$( echo $DB_SPEC | tr '[a-z]' '[A-Z]')" != "ALL" ]]; then
|
||||
# Expectation is that the database listing will be put into
|
||||
# the given file one database per line
|
||||
get_databases $TMP_DIR $DB_FILE
|
||||
if [[ "$?" -ne 0 ]]; then
|
||||
clean_and_exit 1 "ERROR: Could not get the list of databases to restore."
|
||||
fi
|
||||
|
||||
#check if the requested database is available in the archive
|
||||
database_exists $DB_SPEC
|
||||
if [[ $? -ne 1 ]]; then
|
||||
clean_and_exit 1 "ERROR: Database ${DB_SPEC} does not exist."
|
||||
fi
|
||||
|
||||
echo "Restoring Database $DB_SPEC And Grants"
|
||||
restore_single_db $DB_SPEC $TMP_DIR
|
||||
if [[ "$?" -eq 0 ]]; then
|
||||
echo "Single database restored successfully."
|
||||
else
|
||||
clean_and_exit 1 "ERROR: Single database restore failed."
|
||||
fi
|
||||
echo "Tail ${LOG_FILE} for restore log."
|
||||
clean_and_exit 0 ""
|
||||
else
|
||||
echo "Restoring All The Databases. This could take a few minutes..."
|
||||
restore_all_dbs $TMP_DIR
|
||||
if [[ "$?" -eq 0 ]]; then
|
||||
echo "All databases restored successfully."
|
||||
else
|
||||
clean_and_exit 1 "ERROR: Database restore failed."
|
||||
fi
|
||||
clean_and_exit 0 "Tail ${LOG_FILE} for restore log."
|
||||
fi
|
||||
fi
|
||||
else
|
||||
usage 1
|
||||
fi
|
||||
|
||||
clean_and_exit 0 "Done"
|
||||
}
|
||||
{{- end }}
|
||||
@ -12,6 +12,9 @@
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# This is needed to get the postgresql admin password
|
||||
# Turn off tracing so the password doesn't get printed.
|
||||
set +x
|
||||
export PGPASSWORD=$(cat /etc/postgresql/admin_user.conf \
|
||||
| grep postgres | awk -F: '{print $5}')
|
||||
|
||||
@ -19,111 +22,45 @@ export PGPASSWORD=$(cat /etc/postgresql/admin_user.conf \
|
||||
# is needed.
|
||||
set -x
|
||||
|
||||
source /tmp/backup_main.sh
|
||||
|
||||
# Export the variables required by the framework
|
||||
# Note: REMOTE_BACKUP_ENABLED and CONTAINER_NAME are already exported
|
||||
export DB_NAMESPACE=${POSTGRESQL_POD_NAMESPACE}
|
||||
export DB_NAME="postgres"
|
||||
export LOCAL_DAYS_TO_KEEP=$POSTGRESQL_LOCAL_BACKUP_DAYS_TO_KEEP
|
||||
export REMOTE_DAYS_TO_KEEP=$POSTGRESQL_REMOTE_BACKUP_DAYS_TO_KEEP
|
||||
export ARCHIVE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${DB_NAMESPACE}/${DB_NAME}/archive
|
||||
|
||||
# This function dumps all database files to the $TMP_DIR that is being
|
||||
# used as a staging area for preparing the backup tarball. Log file to
|
||||
# write to is passed in - the framework will expect that file to have any
|
||||
# errors that occur if the database dump is unsuccessful, so that it can
|
||||
# add the file contents to its own logs.
|
||||
dump_databases_to_directory() {
|
||||
TMP_DIR=$1
|
||||
LOG_FILE=$2
|
||||
|
||||
PG_DUMPALL_OPTIONS=$POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS
|
||||
TMP_DIR=/tmp/pg_backup
|
||||
BACKUPS_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/current
|
||||
ARCHIVE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/archive
|
||||
LOG_FILE=/tmp/dberror.log
|
||||
PG_DUMPALL="pg_dumpall \
|
||||
$POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS \
|
||||
-U $POSTGRESQL_BACKUP_USER \
|
||||
-U $POSTGRESQL_ADMIN_USER \
|
||||
-h $POSTGRESQL_SERVICE_HOST"
|
||||
|
||||
source /tmp/common_backup_restore.sh
|
||||
|
||||
# Create necessary directories if they do not exist.
|
||||
mkdir -p $BACKUPS_DIR || log_backup_error_exit "Cannot create directory ${BACKUPS_DIR}!"
|
||||
mkdir -p $ARCHIVE_DIR || log_backup_error_exit "Cannot create directory ${ARCHIVE_DIR}!"
|
||||
mkdir -p $TMP_DIR || log_backup_error_exit "Cannot create directory ${TMP_DIR}!"
|
||||
|
||||
# Remove temporary directory contents.
|
||||
rm -rf $BACKUPS_DIR/* || log_backup_error_exit "Cannot clear ${BACKUPS_DIR} directory contents!"
|
||||
rm -rf $TMP_DIR/* || log_backup_error_exit "Cannot clear ${TMP_DIR} directory contents!"
|
||||
|
||||
NOW=$(date +"%Y-%m-%dT%H:%M:%SZ")
|
||||
SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all
|
||||
TARBALL_FILE=${SQL_FILE}.${NOW}.tar.gz
|
||||
|
||||
cd $TMP_DIR || log_backup_error_exit "Cannot change to directory $TMP_DIR"
|
||||
|
||||
rm -f $LOG_FILE
|
||||
cd $TMP_DIR
|
||||
|
||||
#Dump all databases
|
||||
$PG_DUMPALL --file=${TMP_DIR}/${SQL_FILE}.sql 2>>$LOG_FILE
|
||||
if [[ $? -eq 0 && -s "${TMP_DIR}/${SQL_FILE}.sql" ]]
|
||||
then
|
||||
log INFO postgresql_backup "Databases dumped successfully. Creating tarball..."
|
||||
|
||||
#Archive the current database files
|
||||
tar zcvf $ARCHIVE_DIR/$TARBALL_FILE *
|
||||
if [[ $? -ne 0 ]]
|
||||
then
|
||||
log_backup_error_exit "Backup tarball could not be created."
|
||||
fi
|
||||
|
||||
log INFO postgresql_backup "Tarball $TARBALL_FILE created successfully."
|
||||
|
||||
# Remove the sql files as they are no longer needed.
|
||||
rm -rf $TMP_DIR/*
|
||||
|
||||
if {{ .Values.conf.backup.remote_backup.enabled }}
|
||||
then
|
||||
# Copy the tarball back to the BACKUPS_DIR so that the other container
|
||||
# can access it for sending it to remote storage.
|
||||
cp $ARCHIVE_DIR/$TARBALL_FILE $BACKUPS_DIR/$TARBALL_FILE
|
||||
|
||||
if [[ $? -ne 0 ]]
|
||||
then
|
||||
log_backup_error_exit "Backup tarball could not be copied to backup directory ${BACKUPS_DIR}."
|
||||
fi
|
||||
|
||||
# Sleep for a few seconds to allow the file system to get caught up...also to
|
||||
# help prevent race condition where the other container grabs the backup_completed
|
||||
# token and the backup file hasn't completed writing to disk.
|
||||
sleep 30
|
||||
|
||||
# Note: this next line is the trigger that tells the other container to
|
||||
# start sending to remote storage. After this backup is sent to remote
|
||||
# storage, the other container will delete the "current" backup.
|
||||
touch $BACKUPS_DIR/backup_completed
|
||||
if [[ $? -eq 0 && -s "${TMP_DIR}/${SQL_FILE}.sql" ]]; then
|
||||
log INFO postgresql_backup "Databases dumped successfully."
|
||||
return 0
|
||||
else
|
||||
# Remote backup is not enabled. This is ok; at least we have a local backup.
|
||||
log INFO postgresql_backup "Skipping remote backup, as it is not enabled."
|
||||
fi
|
||||
else
|
||||
cat $LOG_FILE
|
||||
rm $LOG_FILE
|
||||
log_backup_error_exit "Backup of the postgresql database failed and needs attention."
|
||||
log ERROR "Backup of the postgresql database failed and needs attention."
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
#Only delete the old archive after a successful archive
|
||||
if [ "$POSTGRESQL_BACKUP_DAYS_TO_KEEP" -gt 0 ]
|
||||
then
|
||||
log INFO postgresql_backup "Deleting backups older than ${POSTGRESQL_BACKUP_DAYS_TO_KEEP} days"
|
||||
if [ -d $ARCHIVE_DIR ]
|
||||
then
|
||||
for ARCHIVE_FILE in $(ls -1 $ARCHIVE_DIR/*.gz)
|
||||
do
|
||||
ARCHIVE_DATE=$( echo $ARCHIVE_FILE | awk -F/ '{print $NF}' | cut -d'.' -f 4)
|
||||
if [ "$(seconds_difference $ARCHIVE_DATE)" -gt "$(($POSTGRESQL_BACKUP_DAYS_TO_KEEP*86400))" ]
|
||||
then
|
||||
log INFO postgresql_backup "Deleting file $ARCHIVE_FILE."
|
||||
rm -rf $ARCHIVE_FILE
|
||||
if [[ $? -ne 0 ]]
|
||||
fhen
|
||||
rm -rf $BACKUPS_DIR/*
|
||||
log_backup_error_exit "Cannot remove ${ARCHIVE_FILE}"
|
||||
fi
|
||||
else
|
||||
log INFO postgresql_backup "Keeping file ${ARCHIVE_FILE}."
|
||||
fi
|
||||
done
|
||||
fi
|
||||
fi
|
||||
|
||||
# Turn off trace just for a clearer printout of backup status - for manual backups, mainly.
|
||||
set +x
|
||||
echo "=================================================================="
|
||||
echo "Backup successful!"
|
||||
echo "Backup archive name: $TARBALL_FILE"
|
||||
echo "=================================================================="
|
||||
# Call main program to start the database backup
|
||||
backup_databases
|
||||
|
||||
@ -12,359 +12,100 @@
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# Capture the user's command line arguments
|
||||
ARGS=("$@")
|
||||
|
||||
# This is needed to get the postgresql admin password
|
||||
# Note: xtracing should be off so it doesn't print the pw
|
||||
export PGPASSWORD=$(cat /etc/postgresql/admin_user.conf \
|
||||
| grep postgres | awk -F: '{print $5}')
|
||||
|
||||
ARCHIVE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/archive
|
||||
RESTORE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/restore
|
||||
source /tmp/restore_main.sh
|
||||
|
||||
# Export the variables needed by the framework
|
||||
export DB_NAME="postgres"
|
||||
export DB_NAMESPACE=${POSTGRESQL_POD_NAMESPACE}
|
||||
export ARCHIVE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${DB_NAMESPACE}/${DB_NAME}/archive
|
||||
|
||||
# Define variables needed in this file
|
||||
POSTGRESQL_HOST=$(cat /etc/postgresql/admin_user.conf | cut -d: -f 1)
|
||||
LOG_FILE=/tmp/dbrestore.log
|
||||
ARGS=("$@")
|
||||
PSQL="psql -U $POSTGRESQL_BACKUP_USER -h $POSTGRESQL_HOST"
|
||||
export PSQL="psql -U $POSTGRESQL_ADMIN_USER -h $POSTGRESQL_HOST"
|
||||
|
||||
source /tmp/common_backup_restore.sh
|
||||
# Extract all databases from an archive and put them in the requested
|
||||
# file.
|
||||
get_databases() {
|
||||
TMP_DIR=$1
|
||||
DB_FILE=$2
|
||||
|
||||
usage() {
|
||||
ret_val=$1
|
||||
echo "Usage:"
|
||||
echo "Restore command options"
|
||||
echo "============================="
|
||||
echo "help"
|
||||
echo "list_archives [remote]"
|
||||
echo "list_databases <archive_filename> [remote]"
|
||||
echo "restore <archive_filename> <db_specifier> [remote]"
|
||||
echo " where <db_specifier> = <dbname> | ALL"
|
||||
clean_and_exit $ret_val ""
|
||||
SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all.sql
|
||||
if [[ -e $TMP_DIR/$SQL_FILE ]]; then
|
||||
grep 'CREATE DATABASE' $TMP_DIR/$SQL_FILE | awk '{ print $3 }' > $DB_FILE
|
||||
else
|
||||
# no databases - just touch the file
|
||||
touch $DB_FILE
|
||||
fi
|
||||
}
|
||||
|
||||
# Extract Single Database SQL Dump from pg_dumpall dump file
|
||||
extract_single_db_dump() {
|
||||
sed "/connect.*$2/,\$!d" $1 | sed "/PostgreSQL database dump complete/,\$d" > \
|
||||
${RESTORE_DIR}/$2.sql
|
||||
sed "/connect.*$2/,\$!d" $1 | sed "/PostgreSQL database dump complete/,\$d" > ${3}/$2.sql
|
||||
}
|
||||
|
||||
#Exit cleanly with some message and return code
|
||||
clean_and_exit() {
|
||||
RETCODE=$1
|
||||
MSG=$2
|
||||
|
||||
#Cleanup Restore Directory
|
||||
rm -rf $RESTORE_DIR/*
|
||||
|
||||
if [[ "x${MSG}" != "x" ]];
|
||||
then
|
||||
echo $MSG
|
||||
fi
|
||||
exit $RETCODE
|
||||
}
|
||||
|
||||
# Signal the other container that it should retrieve a list of archives
|
||||
# from the RGW.
|
||||
retrieve_remote_listing() {
|
||||
# Remove the last response, if there was any
|
||||
rm -rf $RESTORE_DIR/archive_list_*
|
||||
|
||||
# Signal by creating a file in the restore directory
|
||||
touch $RESTORE_DIR/archive_listing_request
|
||||
|
||||
# Wait until the archive listing has been retrieved from the other container.
|
||||
echo "Waiting for archive listing..."
|
||||
wait_for_file $RESTORE_DIR/archive_list_*
|
||||
|
||||
if [[ $? -eq 1 ]]
|
||||
then
|
||||
clean_and_exit 1 "Request failed - container did not respond. Archive listing is NOT available."
|
||||
fi
|
||||
|
||||
ERR=$(cat $RESTORE_DIR/archive_list_error 2>/dev/null)
|
||||
if [[ $? -eq 0 ]]
|
||||
then
|
||||
clean_and_exit 1 "Request failed - ${ERR}"
|
||||
fi
|
||||
|
||||
echo "Done waiting. Archive list is available."
|
||||
}
|
||||
|
||||
# Signal the other container that it should retrieve a single archive
|
||||
# from the RGW.
|
||||
retrieve_remote_archive() {
|
||||
ARCHIVE=$1
|
||||
|
||||
# Remove the last response, if there was any
|
||||
rm -rf $RESTORE_DIR/archive_*
|
||||
|
||||
# Signal by creating a file in the restore directory containing the archive
|
||||
# name.
|
||||
echo "$ARCHIVE" > $RESTORE_DIR/get_archive_request
|
||||
|
||||
# Wait until the archive has been retrieved from the other container.
|
||||
echo "Waiting for requested archive ${ARCHIVE}..."
|
||||
wait_for_file $RESTORE_DIR/archive_*
|
||||
|
||||
if [[ $? -eq 1 ]]
|
||||
then
|
||||
clean_and_exit 1 "Request failed - container did not respond. Archive ${ARCHIVE} is NOT available."
|
||||
fi
|
||||
|
||||
ERR=$(cat $RESTORE_DIR/archive_error 2>/dev/null)
|
||||
if [[ $? -eq 0 ]]
|
||||
then
|
||||
clean_and_exit 1 "Request failed - ${ERR}"
|
||||
fi
|
||||
|
||||
rm -rf $RESTORE_DIR/archive_response
|
||||
if [[ -e $RESTORE_DIR/$ARCHIVE ]]
|
||||
then
|
||||
echo "Done waiting. Archive $ARCHIVE is available."
|
||||
else
|
||||
clean_and_exit 1 "Request failed - Archive $ARCHIVE is NOT available."
|
||||
fi
|
||||
}
|
||||
|
||||
#Display all archives
|
||||
list_archives() {
|
||||
REMOTE=$1
|
||||
|
||||
if [[ "x${REMOTE^^}" == "xREMOTE" ]]
|
||||
then
|
||||
retrieve_remote_listing
|
||||
if [[ -e $RESTORE_DIR/archive_list_response ]]
|
||||
then
|
||||
echo
|
||||
echo "All Archives from RGW Data Store"
|
||||
echo "=============================================="
|
||||
cat $RESTORE_DIR/archive_list_response
|
||||
clean_and_exit 0 ""
|
||||
else
|
||||
clean_and_exit 1 "Archives could not be retrieved from the RGW."
|
||||
fi
|
||||
elif [[ "x${REMOTE}" == "x" ]]
|
||||
then
|
||||
if [ -d $ARCHIVE_DIR ]
|
||||
then
|
||||
archives=$(find $ARCHIVE_DIR/ -iname "*.gz" -print)
|
||||
echo
|
||||
echo "All Local Archives"
|
||||
echo "=============================================="
|
||||
for archive in $archives
|
||||
do
|
||||
echo $archive | cut -d '/' -f 8
|
||||
done
|
||||
clean_and_exit 0 ""
|
||||
else
|
||||
clean_and_exit 1 "Local archive directory is not available."
|
||||
fi
|
||||
else
|
||||
usage 1
|
||||
fi
|
||||
}
|
||||
|
||||
#Return all databases from an archive
|
||||
get_databases() {
|
||||
ARCHIVE_FILE=$1
|
||||
REMOTE=$2
|
||||
|
||||
if [[ "x$REMOTE" == "xremote" ]]
|
||||
then
|
||||
retrieve_remote_archive $ARCHIVE_FILE
|
||||
elif [[ "x$REMOTE" == "x" ]]
|
||||
then
|
||||
if [ -e $ARCHIVE_DIR/$ARCHIVE_FILE ]
|
||||
then
|
||||
cp $ARCHIVE_DIR/$ARCHIVE_FILE $RESTORE_DIR/$ARCHIVE_FILE
|
||||
if [[ $? != 0 ]]
|
||||
then
|
||||
clean_and_exit 1 "Could not copy local archive to restore directory."
|
||||
fi
|
||||
else
|
||||
clean_and_exit 1 "Local archive file could not be found."
|
||||
fi
|
||||
else
|
||||
usage 1
|
||||
fi
|
||||
|
||||
echo "Decompressing archive $ARCHIVE_FILE..."
|
||||
cd $RESTORE_DIR
|
||||
tar zxvf - < $RESTORE_DIR/$ARCHIVE_FILE 1>/dev/null
|
||||
SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all.sql
|
||||
if [ -e $RESTORE_DIR/$SQL_FILE ]
|
||||
then
|
||||
DBS=$( grep 'CREATE DATABASE' $RESTORE_DIR/$SQL_FILE | awk '{ print $3 }' )
|
||||
else
|
||||
DBS=" "
|
||||
fi
|
||||
}
|
||||
|
||||
#Display all databases from an archive
|
||||
list_databases() {
|
||||
ARCHIVE_FILE=$1
|
||||
REMOTE=$2
|
||||
WHERE="local"
|
||||
|
||||
if [[ "x${REMOTE}" != "x" ]]
|
||||
then
|
||||
WHERE="remote"
|
||||
fi
|
||||
|
||||
get_databases $ARCHIVE_FILE $REMOTE
|
||||
if [ -n "$DBS" ]
|
||||
then
|
||||
echo " "
|
||||
echo "Databases in the $WHERE archive $ARCHIVE_FILE"
|
||||
echo "================================================================================"
|
||||
for db in $DBS
|
||||
do
|
||||
echo $db
|
||||
done
|
||||
else
|
||||
echo "There is no database in the archive."
|
||||
fi
|
||||
}
|
||||
|
||||
create_db_if_not_exist() {
|
||||
#Postgresql does not have the concept of creating
|
||||
#database if condition. This function help create
|
||||
#the database in case it does not exist
|
||||
$PSQL -tc "SELECT 1 FROM pg_database WHERE datname = '$1'" | grep -q 1 || \
|
||||
$PSQL -c "CREATE DATABASE $1"
|
||||
}
|
||||
|
||||
#Restore a single database dump from pg_dumpall dump.
|
||||
# Restore a single database dump from pg_dumpall sql dumpfile.
|
||||
restore_single_db() {
|
||||
SINGLE_DB_NAME=$1
|
||||
if [ -z "$SINGLE_DB_NAME" ]
|
||||
then
|
||||
usage 1
|
||||
fi
|
||||
TMP_DIR=$2
|
||||
|
||||
SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all.sql
|
||||
if [ -f $RESTORE_DIR/$SQL_FILE ]
|
||||
then
|
||||
extract_single_db_dump $RESTORE_DIR/$SQL_FILE $SINGLE_DB_NAME
|
||||
if [[ -f $RESTORE_DIR/$SINGLE_DB_NAME.sql && -s $RESTORE_DIR/$SINGLE_DB_NAME.sql ]]
|
||||
then
|
||||
create_db_if_not_exist $single_db_name
|
||||
$PSQL -d $SINGLE_DB_NAME -f ${RESTORE_DIR}/${SINGLE_DB_NAME}.sql 2>>$LOG_FILE >> $LOG_FILE
|
||||
if [ "$?" -eq 0 ]
|
||||
then
|
||||
echo "Database Restore Successful."
|
||||
if [[ -f $TMP_DIR/$SQL_FILE ]]; then
|
||||
extract_single_db_dump $TMP_DIR/$SQL_FILE $SINGLE_DB_NAME $TMP_DIR
|
||||
if [[ -f $TMP_DIR/$SINGLE_DB_NAME.sql && -s $TMP_DIR/$SINGLE_DB_NAME.sql ]]; then
|
||||
# Postgresql does not have the concept of creating database if condition.
|
||||
# This next command creates the database in case it does not exist.
|
||||
$PSQL -tc "SELECT 1 FROM pg_database WHERE datname = '$SINGLE_DB_NAME'" | grep -q 1 || \
|
||||
$PSQL -c "CREATE DATABASE $SINGLE_DB_NAME"
|
||||
if [[ "$?" -ne 0 ]]; then
|
||||
echo "Could not create the single database being restored: ${SINGLE_DB_NAME}."
|
||||
return 1
|
||||
fi
|
||||
$PSQL -d $SINGLE_DB_NAME -f ${TMP_DIR}/${SINGLE_DB_NAME}.sql 2>>$LOG_FILE >> $LOG_FILE
|
||||
if [[ "$?" -eq 0 ]]; then
|
||||
echo "Database restore Successful."
|
||||
else
|
||||
clean_and_exit 1 "Database Restore Failed."
|
||||
echo "Database restore Failed."
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
clean_and_exit 1 "Database Dump For $SINGLE_DB_NAME is empty or not available."
|
||||
echo "Database dump For $SINGLE_DB_NAME is empty or not available."
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
clean_and_exit 1 "Database file for dump_all not available to restore from"
|
||||
echo "No database file available to restore from."
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
#Restore all the databases
|
||||
# Restore all the databases from the pg_dumpall sql file.
|
||||
restore_all_dbs() {
|
||||
TMP_DIR=$1
|
||||
|
||||
SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all.sql
|
||||
if [ -f $RESTORE_DIR/$SQL_FILE ]
|
||||
then
|
||||
$PSQL postgres -f $RESTORE_DIR/$SQL_FILE 2>>$LOG_FILE >> $LOG_FILE
|
||||
if [ "$?" -eq 0 ]
|
||||
then
|
||||
if [[ -f $TMP_DIR/$SQL_FILE ]]; then
|
||||
$PSQL postgres -f $TMP_DIR/$SQL_FILE 2>>$LOG_FILE >> $LOG_FILE
|
||||
if [[ "$?" -eq 0 ]]; then
|
||||
echo "Database Restore successful."
|
||||
else
|
||||
clean_and_exit 1 "Database Restore failed."
|
||||
echo "Database Restore failed."
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
clean_and_exit 1 "There is no database file available to restore from"
|
||||
echo "There is no database file available to restore from."
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
|
||||
is_Option() {
|
||||
opts=$1
|
||||
param=$2
|
||||
find=0
|
||||
for opt in $opts
|
||||
do
|
||||
if [ "$opt" == "$param" ]
|
||||
then
|
||||
find=1
|
||||
fi
|
||||
done
|
||||
echo $find
|
||||
}
|
||||
|
||||
#Main
|
||||
#Create Restore Directory if it's not created already
|
||||
mkdir -p $RESTORE_DIR
|
||||
|
||||
#Cleanup Restore Directory
|
||||
rm -rf $RESTORE_DIR/*
|
||||
|
||||
if [ ${#ARGS[@]} -gt 4 ]
|
||||
then
|
||||
usage 1
|
||||
elif [ ${#ARGS[@]} -eq 1 ]
|
||||
then
|
||||
if [ "${ARGS[0]}" == "list_archives" ]
|
||||
then
|
||||
list_archives
|
||||
clean_and_exit 0 ""
|
||||
elif [ "${ARGS[0]}" == "help" ]
|
||||
then
|
||||
usage 0
|
||||
else
|
||||
usage 1
|
||||
fi
|
||||
elif [ ${#ARGS[@]} -eq 2 ]
|
||||
then
|
||||
if [ "${ARGS[0]}" == "list_databases" ]
|
||||
then
|
||||
list_databases ${ARGS[1]}
|
||||
clean_and_exit 0 ""
|
||||
elif [ "${ARGS[0]}" == "list_archives" ]
|
||||
then
|
||||
list_archives ${ARGS[1]}
|
||||
clean_and_exit 0 ""
|
||||
else
|
||||
usage 1
|
||||
fi
|
||||
elif [[ ${#ARGS[@]} -eq 3 ]] || [[ ${#ARGS[@]} -eq 4 ]]
|
||||
then
|
||||
if [ "${ARGS[0]}" == "list_databases" ]
|
||||
then
|
||||
list_databases ${ARGS[1]} ${ARGS[2]}
|
||||
clean_and_exit 0 ""
|
||||
elif [ "${ARGS[0]}" != "restore" ]
|
||||
then
|
||||
usage 1
|
||||
else
|
||||
ARCHIVE=${ARGS[1]}
|
||||
DB_SPEC=${ARGS[2]}
|
||||
REMOTE=""
|
||||
if [ ${#ARGS[@]} -eq 4 ]
|
||||
then
|
||||
REMOTE=${ARGS[3]}
|
||||
fi
|
||||
|
||||
#Get all the databases in that archive
|
||||
get_databases $ARCHIVE $REMOTE
|
||||
|
||||
#check if the requested database is available in the archive
|
||||
if [ $(is_Option "$DBS" $DB_SPEC) -eq 1 ]
|
||||
then
|
||||
echo "Restoring Database $DB_SPEC And Grants"
|
||||
restore_single_db $DB_SPEC
|
||||
echo "Tail ${LOG_FILE} for restore log."
|
||||
clean_and_exit 0 ""
|
||||
elif [ "$( echo $DB_SPEC | tr '[a-z]' '[A-Z]')" == "ALL" ]
|
||||
then
|
||||
echo "Restoring All The Databases. This could take a few minutes..."
|
||||
restore_all_dbs
|
||||
clean_and_exit 0 "Tail ${LOG_FILE} for restore log."
|
||||
else
|
||||
clean_and_exit 1 "There is no database with that name"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
usage 1
|
||||
fi
|
||||
|
||||
clean_and_exit 0 "Done"
|
||||
# Call the CLI interpreter, providing the archive directory path and the
|
||||
# user arguments passed in
|
||||
cli_main ${ARGS[@]}
|
||||
|
||||
@ -32,9 +32,8 @@ data:
|
||||
{{- if .Values.conf.backup.enabled }}
|
||||
backup_postgresql.sh: {{ tuple "bin/_backup_postgresql.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }}
|
||||
restore_postgresql.sh: {{ tuple "bin/_restore_postgresql.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }}
|
||||
remote_store_postgresql.sh: {{ tuple "bin/_remote_store_postgresql.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }}
|
||||
remote_retrieve_postgresql.sh: {{ tuple "bin/_remote_retrieve_postgresql.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }}
|
||||
common_backup_restore.sh: {{ tuple "bin/_common_backup_restore.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }}
|
||||
backup_main.sh: {{ include "helm-toolkit.scripts.db-backup-restore.backup_main" . | b64enc }}
|
||||
restore_main.sh: {{ include "helm-toolkit.scripts.db-backup-restore.restore_main" . | b64enc }}
|
||||
{{- end }}
|
||||
{{- if .Values.manifests.job_ks_user }}
|
||||
ks-user.sh: {{ include "helm-toolkit.scripts.keystone_user" . | b64enc }}
|
||||
|
||||
@ -48,6 +48,7 @@ spec:
|
||||
{{ tuple $envAll "postgresql-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 12 }}
|
||||
spec:
|
||||
securityContext:
|
||||
runAsUser: 65534
|
||||
fsGroup: 999
|
||||
serviceAccountName: {{ $serviceAccountName }}
|
||||
restartPolicy: OnFailure
|
||||
@ -55,17 +56,17 @@ spec:
|
||||
{{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
|
||||
containers:
|
||||
- name: postgresql-backup
|
||||
{{ tuple $envAll "postgresql" | include "helm-toolkit.snippets.image" | indent 14 }}
|
||||
{{ tuple $envAll "postgresql_backup" | include "helm-toolkit.snippets.image" | indent 14 }}
|
||||
{{ tuple $envAll $envAll.Values.pod.resources.jobs.postgresql_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }}
|
||||
command:
|
||||
- /tmp/backup_postgresql.sh
|
||||
env:
|
||||
- name: POSTGRESQL_BACKUP_PASSWORD
|
||||
- name: POSTGRESQL_ADMIN_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
key: POSTGRES_PASSWORD
|
||||
name: postgresql-admin
|
||||
- name: POSTGRESQL_BACKUP_USER
|
||||
- name: POSTGRESQL_ADMIN_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
key: POSTGRES_USER
|
||||
@ -74,56 +75,42 @@ spec:
|
||||
value: {{ .Values.conf.backup.base_path }}
|
||||
- name: POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS
|
||||
value: {{ .Values.conf.backup.pg_dumpall_options }}
|
||||
- name: POSTGRESQL_BACKUP_DAYS_TO_KEEP
|
||||
value: "{{ .Values.conf.backup.days_of_backup_to_keep }}"
|
||||
- name: POSTGRESQL_LOCAL_BACKUP_DAYS_TO_KEEP
|
||||
value: "{{ .Values.conf.backup.days_to_keep }}"
|
||||
- name: POSTGRESQL_POD_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
- name: REMOTE_BACKUP_ENABLED
|
||||
value: "{{ .Values.conf.backup.remote_backup.enabled }}"
|
||||
{{- if .Values.conf.backup.remote_backup.enabled }}
|
||||
- name: POSTGRESQL_REMOTE_BACKUP_DAYS_TO_KEEP
|
||||
value: "{{ .Values.conf.backup.remote_backup.days_to_keep }}"
|
||||
- name: CONTAINER_NAME
|
||||
value: "{{ .Values.conf.backup.remote_backup.container_name }}"
|
||||
- name: STORAGE_POLICY
|
||||
value: "{{ .Values.conf.backup.remote_backup.storage_policy }}"
|
||||
{{- with $env := dict "ksUserSecret" $envAll.Values.secrets.identity.postgresql }}
|
||||
{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 16 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
- name: pod-tmp
|
||||
mountPath: /tmp
|
||||
- mountPath: /tmp/common_backup_restore.sh
|
||||
name: postgresql-bin
|
||||
readOnly: true
|
||||
subPath: common_backup_restore.sh
|
||||
- mountPath: /tmp/backup_postgresql.sh
|
||||
name: postgresql-bin
|
||||
readOnly: true
|
||||
subPath: backup_postgresql.sh
|
||||
- mountPath: /tmp/backup_main.sh
|
||||
name: postgresql-bin
|
||||
readOnly: true
|
||||
subPath: backup_main.sh
|
||||
- mountPath: {{ .Values.conf.backup.base_path }}
|
||||
name: postgresql-backup-dir
|
||||
- name: postgresql-secrets
|
||||
mountPath: /etc/postgresql/admin_user.conf
|
||||
subPath: admin_user.conf
|
||||
readOnly: true
|
||||
- name: postgresql-remote-store
|
||||
{{ tuple $envAll "postgresql_remote_store" | include "helm-toolkit.snippets.image" | indent 14 }}
|
||||
command:
|
||||
- /tmp/remote_store_postgresql.sh
|
||||
env:
|
||||
{{- with $env := dict "ksUserSecret" $envAll.Values.secrets.identity.postgresql }}
|
||||
{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 16 }}
|
||||
{{- end }}
|
||||
- name: POSTGRESQL_BACKUP_BASE_DIR
|
||||
value: {{ .Values.conf.backup.base_path }}
|
||||
- name: POSTGRESQL_BACKUP_DAYS_TO_KEEP
|
||||
value: "{{ .Values.conf.backup.days_of_backup_to_keep }}"
|
||||
- name: POSTGRESQL_POD_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
volumeMounts:
|
||||
- mountPath: /tmp/common_backup_restore.sh
|
||||
name: postgresql-bin
|
||||
readOnly: true
|
||||
subPath: common_backup_restore.sh
|
||||
- mountPath: /tmp/remote_store_postgresql.sh
|
||||
name: postgresql-bin
|
||||
readOnly: true
|
||||
subPath: remote_store_postgresql.sh
|
||||
- mountPath: {{ .Values.conf.backup.base_path }}
|
||||
name: postgresql-backup-dir
|
||||
restartPolicy: OnFailure
|
||||
serviceAccount: {{ $serviceAccountName }}
|
||||
serviceAccountName: {{ $serviceAccountName }}
|
||||
|
||||
@ -20,7 +20,7 @@ metadata:
|
||||
name: {{ $secretName }}
|
||||
type: Opaque
|
||||
data:
|
||||
{{- $identityClass := .Values.endpoints.identity.auth.postgresql }}
|
||||
{{- $identityClass := index .Values.endpoints.identity.auth $userClass }}
|
||||
{{- if $identityClass.auth_url }}
|
||||
OS_AUTH_URL: {{ $identityClass.auth_url | b64enc }}
|
||||
{{- else }}
|
||||
@ -47,7 +47,7 @@ type: Opaque
|
||||
data:
|
||||
{{- $identityClass := index .Values.endpoints.identity.auth $userClass }}
|
||||
{{- if $identityClass.auth_url }}
|
||||
OS_AUTH_URL: {{ $identityClass.auth_url }}
|
||||
OS_AUTH_URL: {{ $identityClass.auth_url | b64enc }}
|
||||
{{- else }}
|
||||
OS_AUTH_URL: {{ tuple "identity" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | b64enc }}
|
||||
{{- end }}
|
||||
|
||||
@ -132,7 +132,7 @@ images:
|
||||
ks_user: docker.io/openstackhelm/heat:stein-ubuntu_bionic
|
||||
prometheus_postgresql_exporter: docker.io/wrouesnel/postgres_exporter:v0.4.6
|
||||
prometheus_postgresql_exporter_create_user: "docker.io/postgres:9.5"
|
||||
postgresql_remote_store: docker.io/openstackhelm/heat:stein-ubuntu_bionic
|
||||
postgresql_backup: "quay.io/airshipit/porthole-postgresql-utility:latest-ubuntu_bionic"
|
||||
pull_policy: "IfNotPresent"
|
||||
local_registry:
|
||||
active: false
|
||||
@ -367,11 +367,14 @@ conf:
|
||||
backup:
|
||||
enabled: false
|
||||
base_path: /var/backup
|
||||
days_of_backup_to_keep: 3
|
||||
days_to_keep: 3
|
||||
pg_dumpall_options: null
|
||||
remote_backup:
|
||||
enabled: false
|
||||
container_name: postgresql
|
||||
days_to_keep: 14
|
||||
storage_policy: default-placement
|
||||
|
||||
exporter:
|
||||
queries:
|
||||
pg_replication:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user