Add capabilitity to backup only a single database

This PS adds the capability to Mariadb and Postgresql to backup a
single database (as an optional parameter to the backup script).

Change-Id: I9bc1eb0173063638b2cf58465c063f602ed20bc1
This commit is contained in:
Parsons, Cliff (cp769u) 2020-08-13 16:44:39 +00:00
parent ba601e0cba
commit 233197fc0b
5 changed files with 126 additions and 66 deletions

View File

@ -9,10 +9,12 @@
# source /tmp/backup_main.sh
#
# Then the script should call the main backup function (backup_databases):
# backup_databases
# backup_databases [scope]
# [scope] is an optional parameter, defaulted to "all". If only one specific
# database is required to be backed up then this parameter will
# contain the name of the database; otherwise all are backed up.
#
# No arguments required. However, the framework will require the
# following variables to be exported:
# The framework will require the following variables to be exported:
#
# export DB_NAMESPACE Namespace where the database(s) reside
# export DB_NAME Name of the database system
@ -44,12 +46,15 @@
# is 1800 (30 minutes).
#
# The database-specific functions that need to be implemented are:
# dump_databases_to_directory <directory> <err_logfile>
# dump_databases_to_directory <directory> <err_logfile> [scope]
# where:
# <directory> is the full directory path to dump the database files
# into. This is a temporary directory for this backup only.
# <err_logfile> is the full directory path where error logs are to be
# written by the application.
# [scope] set to "all" if all databases are to be backed up; or
# set to the name of a specific database to be backed up.
# This optional parameter is defaulted to "all".
# returns: 0 if no errors; 1 if any errors occurred
#
# This function is expected to dump the database file(s) to the specified
@ -285,7 +290,11 @@ remove_old_remote_archives() {
# 1) The directory where the final backup will be kept after it is compressed.
# 2) A temporary directory to use for placing database files to be compressed.
# Note: this temp directory will be deleted after backup is done.
# 3) Optional "scope" parameter indicating what database to back up. Defaults
# to "all".
backup_databases() {
SCOPE=${1:-"all"}
# Create necessary directories if they do not exist.
mkdir -p $ARCHIVE_DIR || log_backup_error_exit "Cannot create directory ${ARCHIVE_DIR}!"
export TMP_DIR=$(mktemp -d) || log_backup_error_exit "Cannot create temp directory!"
@ -294,7 +303,7 @@ backup_databases() {
export ERR_LOG_FILE=$(mktemp -p /tmp) || log_backup_error_exit "Cannot create log file!"
# It is expected that this function will dump the database files to the $TMP_DIR
dump_databases_to_directory $TMP_DIR $ERR_LOG_FILE
dump_databases_to_directory $TMP_DIR $ERR_LOG_FILE $SCOPE
# If successful, there should be at least one file in the TMP_DIR
if [[ $? -ne 0 || $(ls $TMP_DIR | wc -w) -eq 0 ]]; then
@ -305,7 +314,7 @@ backup_databases() {
log INFO "${DB_NAME}_backup" "Databases dumped successfully. Creating tarball..."
NOW=$(date +"%Y-%m-%dT%H:%M:%SZ")
TARBALL_FILE="${DB_NAME}.${DB_NAMESPACE}.all.${NOW}.tar.gz"
TARBALL_FILE="${DB_NAME}.${DB_NAMESPACE}.${SCOPE}.${NOW}.tar.gz"
cd $TMP_DIR || log_backup_error_exit "Cannot change to directory $TMP_DIR"

View File

@ -1,5 +1,7 @@
#!/bin/bash
SCOPE=${1:-"all"}
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
@ -28,6 +30,7 @@ export ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${DB_NAMESPACE}/${DB_NAME}/arch
dump_databases_to_directory() {
TMP_DIR=$1
LOG_FILE=$2
SCOPE=${3:-"all"}
MYSQL="mysql \
--defaults-file=/etc/mysql/admin_user.cnf \
@ -36,9 +39,18 @@ dump_databases_to_directory() {
MYSQLDUMP="mysqldump \
--defaults-file=/etc/mysql/admin_user.cnf"
MYSQL_DBNAMES=( $($MYSQL --silent --skip-column-names -e \
"show databases;" | \
egrep -vi 'information_schema|performance_schema|mysql') )
if [[ "${SCOPE}" == "all" ]]; then
MYSQL_DBNAMES=( $($MYSQL --silent --skip-column-names -e \
"show databases;" | \
egrep -vi 'information_schema|performance_schema|mysql') )
else
if [[ "${SCOPE}" != "information_schema" && "${SCOPE}" != "performance_schema" && "${SCOPE}" != "mysql" ]]; then
MYSQL_DBNAMES=( ${SCOPE} )
else
log ERROR "It is not allowed to backup database ${SCOPE}."
return 1
fi
fi
#check if there is a database to backup, otherwise exit
if [[ -z "${MYSQL_DBNAMES// }" ]]
@ -50,19 +62,21 @@ dump_databases_to_directory() {
#Create a list of Databases
printf "%s\n" "${MYSQL_DBNAMES[@]}" > $TMP_DIR/db.list
#Retrieve and create the GRANT file for all the users
if [[ "${SCOPE}" == "all" ]]; then
#Retrieve and create the GRANT file for all the users
{{- if .Values.manifests.certificates }}
SSL_DSN=";mysql_ssl=1"
SSL_DSN="$SSL_DSN;mysql_ssl_client_key=/etc/mysql/certs/tls.key"
SSL_DSN="$SSL_DSN;mysql_ssl_client_cert=/etc/mysql/certs/tls.crt"
SSL_DSN="$SSL_DSN;mysql_ssl_ca_file=/etc/mysql/certs/ca.crt"
if ! pt-show-grants --defaults-file=/etc/mysql/admin_user.cnf $SSL_DSN \
SSL_DSN=";mysql_ssl=1"
SSL_DSN="$SSL_DSN;mysql_ssl_client_key=/etc/mysql/certs/tls.key"
SSL_DSN="$SSL_DSN;mysql_ssl_client_cert=/etc/mysql/certs/tls.crt"
SSL_DSN="$SSL_DSN;mysql_ssl_ca_file=/etc/mysql/certs/ca.crt"
if ! pt-show-grants --defaults-file=/etc/mysql/admin_user.cnf $SSL_DSN \
{{- else }}
if ! pt-show-grants --defaults-file=/etc/mysql/admin_user.cnf \
if ! pt-show-grants --defaults-file=/etc/mysql/admin_user.cnf \
{{- end }}
2>>"$LOG_FILE" > "$TMP_DIR"/grants.sql; then
log ERROR "Failed to create GRANT for all the users"
return 1
2>>"$LOG_FILE" > "$TMP_DIR"/grants.sql; then
log ERROR "Failed to create GRANT for all the users"
return 1
fi
fi
#Retrieve and create the GRANT files per DB
@ -82,22 +96,20 @@ dump_databases_to_directory() {
done
#Dumping the database
DATE=$(date +'%Y-%m-%dT%H:%M:%SZ')
SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all
TARBALL_FILE=${SQL_FILE}.${DATE}.tar.gz
SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.${SCOPE}
$MYSQLDUMP $MYSQL_BACKUP_MYSQLDUMP_OPTIONS "${MYSQL_DBNAMES[@]}" \
> $TMP_DIR/${SQL_FILE}.sql 2>>$LOG_FILE
if [[ $? -eq 0 && -s $TMP_DIR/${SQL_FILE}.sql ]]
then
log INFO "Databases dumped successfully."
log INFO "Database(s) dumped successfully. (SCOPE = ${SCOPE})"
return 0
else
log ERROR "Backup failed and need attention."
log ERROR "Backup failed and need attention. (SCOPE = ${SCOPE})"
return 1
fi
}
# Call main program to start the database backup
backup_databases
backup_databases ${SCOPE}

View File

@ -83,9 +83,9 @@ get_tables() {
TMP_DIR=$2
TABLE_FILE=$3
SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql
if [[ -e $TMP_DIR/$SQL_FILE ]]; then
current_db_desc ${DATABASE} ${TMP_DIR}/${SQL_FILE} \
SQL_FILE=$TMP_DIR/mariadb.$MARIADB_POD_NAMESPACE.*.sql
if [ -f $SQL_FILE ]; then
current_db_desc ${DATABASE} ${SQL_FILE} \
| grep "^CREATE TABLE" | awk -F '`' '{print $2}' \
> $TABLE_FILE
else
@ -103,9 +103,9 @@ get_rows() {
TMP_DIR=$3
ROW_FILE=$4
SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql
if [[ -e $TMP_DIR/$SQL_FILE ]]; then
current_db_desc ${DATABASE} ${TMP_DIR}/${SQL_FILE} \
SQL_FILE=$TMP_DIR/mariadb.$MARIADB_POD_NAMESPACE.*.sql
if [ -f $SQL_FILE ]; then
current_db_desc ${DATABASE} ${SQL_FILE} \
| grep "INSERT INTO \`${TABLE}\` VALUES" > $ROW_FILE
return 0
else
@ -123,10 +123,10 @@ get_schema() {
TMP_DIR=$3
SCHEMA_FILE=$4
SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql
if [[ -e $TMP_DIR/$SQL_FILE ]]; then
SQL_FILE=$TMP_DIR/mariadb.$MARIADB_POD_NAMESPACE.*.sql
if [ -f $SQL_FILE ]; then
DB_FILE=$(mktemp -p /tmp)
current_db_desc ${DATABASE} ${TMP_DIR}/${SQL_FILE} > ${DB_FILE}
current_db_desc ${DATABASE} ${SQL_FILE} > ${DB_FILE}
sed -n /'CREATE TABLE `'$TABLE'`'/,/'--'/p ${DB_FILE} > ${SCHEMA_FILE}
if [[ ! (-s ${SCHEMA_FILE}) ]]; then
sed -n /'CREATE TABLE IF NOT EXISTS `'$TABLE'`'/,/'--'/p ${DB_FILE} \
@ -193,8 +193,8 @@ restore_single_db() {
return 1
fi
SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql
if [[ -f ${TMP_DIR}/$SQL_FILE ]]
SQL_FILE=$TMP_DIR/mariadb.$MARIADB_POD_NAMESPACE.*.sql
if [ -f $SQL_FILE ]
then
# Restoring a single database requires us to create a temporary user
# which has capability to only restore that ONE database. One gotcha
@ -208,7 +208,7 @@ restore_single_db() {
echo "Restore $SINGLE_DB_NAME failed create restore user."
return 1
fi
$RESTORE_CMD --force < ${TMP_DIR}/$SQL_FILE 2>>$RESTORE_LOG
$RESTORE_CMD --force < $SQL_FILE 2>>$RESTORE_LOG
if [[ "$?" -eq 0 ]]
then
echo "Database $SINGLE_DB_NAME Restore successful."
@ -254,10 +254,20 @@ restore_single_db() {
restore_all_dbs() {
TMP_DIR=$1
SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql
if [[ -f ${TMP_DIR}/$SQL_FILE ]]
SQL_FILE=$TMP_DIR/mariadb.$MARIADB_POD_NAMESPACE.*.sql
if [ -f $SQL_FILE ]
then
$MYSQL < ${TMP_DIR}/$SQL_FILE 2>$RESTORE_LOG
# Check the scope of the archive.
SCOPE=$(echo ${SQL_FILE} | awk -F'.' '{print $(NF-1)}')
if [[ "${SCOPE}" != "all" ]]; then
# This is just a single database backup. The user should
# instead use the single database restore option.
echo "Cannot use the restore all option for an archive containing only a single database."
echo "Please use the single database restore option."
return 1
fi
$MYSQL < $SQL_FILE 2>$RESTORE_LOG
if [[ "$?" -eq 0 ]]
then
echo "Databases $( echo $DBS | tr -d '\n') Restore successful."

View File

@ -1,5 +1,7 @@
#!/bin/bash
SCOPE=${1:-"all"}
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
@ -40,27 +42,43 @@ export ARCHIVE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${DB_NAMESPACE}/${DB_NAME}/a
dump_databases_to_directory() {
TMP_DIR=$1
LOG_FILE=$2
SCOPE=${3:-"all"}
PG_DUMPALL_OPTIONS=$(echo $POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS | sed 's/"//g')
PG_DUMP_OPTIONS=$(echo $POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS | sed 's/"//g')
PG_DUMP="pg_dump \
$PG_DUMP_OPTIONS --create \
-U $POSTGRESQL_ADMIN_USER \
-h $POSTGRESQL_SERVICE_HOST"
PG_DUMPALL="pg_dumpall \
$PG_DUMPALL_OPTIONS \
$PG_DUMP_OPTIONS \
-U $POSTGRESQL_ADMIN_USER \
-h $POSTGRESQL_SERVICE_HOST"
SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all
SQL_FILE=postgres.${POSTGRESQL_POD_NAMESPACE}.${SCOPE}
cd $TMP_DIR
#Dump all databases
$PG_DUMPALL --file=${TMP_DIR}/${SQL_FILE}.sql 2>>$LOG_FILE
if [[ "${SCOPE}" == "all" ]]; then
# Dump all databases
${PG_DUMPALL} --file=${TMP_DIR}/${SQL_FILE}.sql 2>>${LOG_FILE}
else
if [[ "${SCOPE}" != "postgres" && "${SCOPE}" != "template0" && "${SCOPE}" != "template1" ]]; then
# Dump the specified database
${PG_DUMP} --file=${TMP_DIR}/${SQL_FILE}.sql ${SCOPE} 2>>${LOG_FILE}
else
log ERROR "It is not allowed to backup up the ${SCOPE} database."
return 1
fi
fi
if [[ $? -eq 0 && -s "${TMP_DIR}/${SQL_FILE}.sql" ]]; then
log INFO postgresql_backup "Databases dumped successfully."
log INFO postgresql_backup "Database(s) dumped successfully. (SCOPE = ${SCOPE})"
return 0
else
log ERROR "Backup of the postgresql database failed and needs attention."
log ERROR "Backup of the postgresql database(s) failed and needs attention. (SCOPE = ${SCOPE})"
return 1
fi
}
# Call main program to start the database backup
backup_databases
backup_databases ${SCOPE}

View File

@ -38,9 +38,9 @@ get_databases() {
TMP_DIR=$1
DB_FILE=$2
SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all.sql
if [[ -e $TMP_DIR/$SQL_FILE ]]; then
grep 'CREATE DATABASE' $TMP_DIR/$SQL_FILE | awk '{ print $3 }' > $DB_FILE
SQL_FILE=$TMP_DIR/postgres.$POSTGRESQL_POD_NAMESPACE.*.sql
if [ -f $SQL_FILE ]; then
grep 'CREATE DATABASE' $SQL_FILE | awk '{ print $3 }' > $DB_FILE
else
# Error, cannot report the databases
echo "No SQL file found - cannot extract the databases"
@ -55,9 +55,9 @@ get_tables() {
TMP_DIR=$2
TABLE_FILE=$3
SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all.sql
if [[ -e $TMP_DIR/$SQL_FILE ]]; then
cat $TMP_DIR/$SQL_FILE | sed -n /'\\connect '$DATABASE/,/'\\connect'/p | grep "CREATE TABLE" | awk -F'[. ]' '{print $4}' > $TABLE_FILE
SQL_FILE=$TMP_DIR/postgres.$POSTGRESQL_POD_NAMESPACE.*.sql
if [ -f $SQL_FILE ]; then
cat $SQL_FILE | sed -n /'\\connect '$DATABASE/,/'\\connect'/p | grep "CREATE TABLE" | awk -F'[. ]' '{print $4}' > $TABLE_FILE
else
# Error, cannot report the tables
echo "No SQL file found - cannot extract the tables"
@ -73,9 +73,9 @@ get_rows() {
TMP_DIR=$3
ROW_FILE=$4
SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all.sql
if [[ -e $TMP_DIR/$SQL_FILE ]]; then
cat $TMP_DIR/$SQL_FILE | sed -n /'\\connect '${DATABASE}/,/'\\connect'/p > /tmp/db.sql
SQL_FILE=$TMP_DIR/postgres.$POSTGRESQL_POD_NAMESPACE.*.sql
if [ -f $SQL_FILE ]; then
cat $SQL_FILE | sed -n /'\\connect '${DATABASE}/,/'\\connect'/p > /tmp/db.sql
cat /tmp/db.sql | grep "INSERT INTO public.${TABLE} VALUES" > $ROW_FILE
rm /tmp/db.sql
else
@ -93,10 +93,10 @@ get_schema() {
TMP_DIR=$3
SCHEMA_FILE=$4
SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all.sql
if [[ -e $TMP_DIR/$SQL_FILE ]]; then
SQL_FILE=$TMP_DIR/postgres.$POSTGRESQL_POD_NAMESPACE.*.sql
if [ -f $SQL_FILE ]; then
DB_FILE=$(mktemp -p /tmp)
cat $TMP_DIR/$SQL_FILE | sed -n /'\\connect '${DATABASE}/,/'\\connect'/p > ${DB_FILE}
cat $SQL_FILE | sed -n /'\\connect '${DATABASE}/,/'\\connect'/p > ${DB_FILE}
cat ${DB_FILE} | sed -n /'CREATE TABLE public.'${TABLE}' ('/,/'--'/p > ${SCHEMA_FILE}
cat ${DB_FILE} | sed -n /'CREATE SEQUENCE public.'${TABLE}/,/'--'/p >> ${SCHEMA_FILE}
cat ${DB_FILE} | sed -n /'ALTER TABLE public.'${TABLE}/,/'--'/p >> ${SCHEMA_FILE}
@ -239,9 +239,9 @@ restore_single_db() {
return 1
fi
SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all.sql
if [[ -f $TMP_DIR/$SQL_FILE ]]; then
extract_single_db_dump $TMP_DIR/$SQL_FILE $SINGLE_DB_NAME $TMP_DIR
SQL_FILE=$TMP_DIR/postgres.$POSTGRESQL_POD_NAMESPACE.*.sql
if [ -f $SQL_FILE ]; then
extract_single_db_dump $SQL_FILE $SINGLE_DB_NAME $TMP_DIR
if [[ -f $TMP_DIR/$SINGLE_DB_NAME.sql && -s $TMP_DIR/$SINGLE_DB_NAME.sql ]]; then
# Drop connections first
drop_connections ${SINGLE_DB_NAME}
@ -308,15 +308,26 @@ restore_all_dbs() {
rm -rf ${LOG_FILE}
touch ${LOG_FILE}
SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all.sql
if [[ -f $TMP_DIR/$SQL_FILE ]]; then
SQL_FILE=$TMP_DIR/postgres.$POSTGRESQL_POD_NAMESPACE.*.sql
if [ -f $SQL_FILE ]; then
# Check the scope of the archive.
SCOPE=$(echo ${SQL_FILE} | awk -F'.' '{print $(NF-1)}')
if [[ "${SCOPE}" != "all" ]]; then
# This is just a single database backup. The user should
# instead use the single database restore option.
echo "Cannot use the restore all option for an archive containing only a single database."
echo "Please use the single database restore option."
return 1
fi
# First drop all connections on all databases
drop_connections_on_all_dbs
if [[ "$?" -ne 0 ]]; then
return 1
fi
$PSQL postgres -f $TMP_DIR/$SQL_FILE 2>>$LOG_FILE >> $LOG_FILE
$PSQL postgres -f $SQL_FILE 2>>$LOG_FILE >> $LOG_FILE
if [[ "$?" -eq 0 ]]; then
if grep "ERROR:" ${LOG_FILE} > /dev/null 2>&1; then
cat ${LOG_FILE}