Mariadb backup/restore enhancements

Below enhancements are made to Mariadb backup:
1) Used new helm-toolkit function to send/retrieve Mariadb
   backups to/from RGW via OpenStack Swift API.
2) Modified the backup script such that the database backup
   tarball can be sent to RGW.
3) Added a keystone user for RGW access.
4) Added a secret for OpenStack Swift API access.
5) Changed the cronjob image and runAsUser
6) Modified the restore script so that archives stored remotely
   on RGW can be used for the restore data source.
7) Added functions to the restore script to retrieve data
   from an archive for tables, table rows and table schema of a databse
8) Added a secret containing all the backup/restore related
   configuration needed for invoking the backup/restore operation
   from a different application or namespace.

Change-Id: Iadb9438fe419cded374897b43337039609077e61

mariadb/templates/bin/_backup_mariadb.sh.tpl

@@ -12,104 +12,76 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 set -x
-BACKUPS_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/mariadb/current
-ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/mariadb/archive
 
-MYSQL="mysql \
+source /tmp/backup_main.sh
-   --defaults-file=/etc/mysql/admin_user.cnf \
-   --connect-timeout 10"
 
-MYSQLDUMP="mysqldump \
+# Export the variables required by the framework
-   --defaults-file=/etc/mysql/admin_user.cnf"
+# Note: REMOTE_BACKUP_ENABLED, STORAGE_POLICY  and CONTAINER_NAME are already
+#       exported.
+export DB_NAMESPACE=${MARIADB_POD_NAMESPACE}
+export DB_NAME="mariadb"
+export LOCAL_DAYS_TO_KEEP=${MARIADB_LOCAL_BACKUP_DAYS_TO_KEEP}
+export REMOTE_DAYS_TO_KEEP=${MARIADB_REMOTE_BACKUP_DAYS_TO_KEEP}
+export ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${DB_NAMESPACE}/${DB_NAME}/archive
 
-seconds_difference() {
+# Dump all the database files to existing $TMP_DIR and save logs to $LOG_FILE
-  archive_date=$( date --date="$1" +%s )
+dump_databases_to_directory() {
-  if [ "$?" -ne 0 ]
+  TMP_DIR=$1
+  LOG_FILE=$2
+
+  MYSQL="mysql \
+     --defaults-file=/etc/mysql/admin_user.cnf \
+     --connect-timeout 10"
+
+  MYSQLDUMP="mysqldump \
+     --defaults-file=/etc/mysql/admin_user.cnf"
+
+  MYSQL_DBNAMES=( $($MYSQL --silent --skip-column-names -e \
+     "show databases;" | \
+     egrep -vi 'information_schema|performance_schema') )
+
+  #check if there is a database to backup, otherwise exit
+  if [[ -z "${MYSQL_DBNAMES// }" ]]
   then
-    second_delta=0
+    log INFO "There is no database to backup"
+    return 0
   fi
-  current_date=$( date +%s )
+
-  second_delta=$(($current_date-$archive_date))
+  #Create a list of Databases
-  if [ "$second_delta" -lt 0 ]
+  printf "%s\n" "${MYSQL_DBNAMES[@]}" > $TMP_DIR/db.list
+
+  #Retrieve and create the GRANT files per DB
+  for db in "${MYSQL_DBNAMES[@]}"
+  do
+    echo $($MYSQL --skip-column-names -e "select concat('show grants for ',user,';') \
+          from mysql.db where ucase(db)=ucase('$db');") | \
+          sed -r "s/show grants for ([a-zA-Z0-9_-]*)/show grants for '\1'/" | \
+          $MYSQL --silent --skip-column-names 2>>$LOG_FILE > $TMP_DIR/${db}_grant.sql
+    if [ "$?" -eq 0 ]
+    then
+      sed -i 's/$/;/' $TMP_DIR/${db}_grant.sql
+    else
+      log ERROR "Failed to create GRANT files for ${db}"
+    fi
+  done
+
+  #Dumping the database
+  DATE=$(date +'%Y-%m-%dT%H:%M:%SZ')
+
+  SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all
+  TARBALL_FILE=${SQL_FILE}.${DATE}.tar.gz
+
+  $MYSQLDUMP $MYSQL_BACKUP_MYSQLDUMP_OPTIONS "${MYSQL_DBNAMES[@]}"  \
+            > $TMP_DIR/${SQL_FILE}.sql 2>>$LOG_FILE
+  if [[ $? -eq 0 && -s $TMP_DIR/${SQL_FILE}.sql ]]
   then
-    second_delta=0
+    log INFO "Databases dumped successfully."
+    return 0
+  else
+    log ERROR "Backup failed and need attention."
+    return 1
   fi
-  echo $second_delta
 }
 
-DBNAME=( $($MYSQL --silent --skip-column-names -e \
+# Call main program to start the database backup
-   "show databases;" | \
+backup_databases
-   egrep -vi 'information_schema|performance_schema|mysql') )
-
-#check if there is a database to backup, otherwise exit
-if [[ -z "${DBNAME// }" ]]
-then
-  echo "There is no database to backup"
-  exit 0
-fi
-
-#Create archive and backup directories.
-mkdir -p $BACKUPS_DIR $ARCHIVE_DIR
-
-#Create a list of Databases
-printf "%s\n" "${DBNAME[@]}" > $BACKUPS_DIR/db.list
-
-#Retrieve and create the GRANT files per DB
-for db in "${DBNAME[@]}"
-do
-  echo $($MYSQL --skip-column-names -e "select concat('show grants for ',user,';') \
-        from mysql.db where ucase(db)=ucase('$db');") | \
-        sed -r "s/show grants for ([a-zA-Z0-9_-]*)/show grants for '\1'/" | \
-        $MYSQL --silent --skip-column-names 2>grant_err.log > $BACKUPS_DIR/${db}_grant.sql
-  if [ "$?" -eq 0 ]
-  then
-    sed -i 's/$/;/' $BACKUPS_DIR/${db}_grant.sql
-  else
-    cat grant_err.log
-  fi
-done
-
-#Dumping the database
-#DATE=$(date +"%Y_%m_%d_%H_%M_%S")
-DATE=$(date +'%Y-%m-%dT%H:%M:%SZ')
-$MYSQLDUMP $MYSQL_BACKUP_MYSQLDUMP_OPTIONS "${DBNAME[@]}"  \
-          > $BACKUPS_DIR/mariadb.all.sql 2>dberror.log
-if [[ $? -eq 0 && -s $BACKUPS_DIR/mariadb.all.sql ]]
-then
-  #Archive the current db files
-  pushd $BACKUPS_DIR 1>/dev/null
-  tar zcvf $ARCHIVE_DIR/mariadb.all.${DATE}.tar.gz *
-  ARCHIVE_RET=$?
-  popd 1>/dev/null
-else
-  #TODO: This can be convert into mail alert of alert send to a monitoring system
-  echo "Backup failed and need attention."
-  cat dberror.log
-  exit 1
-fi
-
-#Remove the current backup
-if [ -d $BACKUPS_DIR ]
-then
-  rm -rf $BACKUPS_DIR/*.sql
-fi
-
-#Only delete the old archive after a successful archive
-if [ $ARCHIVE_RET -eq 0 ]
-  then
-    if [ "$MARIADB_BACKUP_DAYS_TO_KEEP" -gt 0 ]
-    then
-      echo "Deleting backups older than $MARIADB_BACKUP_DAYS_TO_KEEP days"
-      if [ -d $ARCHIVE_DIR ]
-      then
-        for archive_file in $(ls -1 $ARCHIVE_DIR/*.gz)
-        do
-          archive_date=$( echo $archive_file | awk -F/ '{print $NF}' | cut -d'.' -f 3)
-          if [ "$(seconds_difference $archive_date)" -gt "$(($MARIADB_BACKUP_DAYS_TO_KEEP*86400))" ]
-          then
-            rm -rf $archive_file
-          fi
-        done
-      fi
-    fi
-fi

mariadb/templates/bin/_restore_mariadb.sh.tpl

@@ -12,22 +12,26 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-log_error() {
+# Capture the user's command line arguments
-  echo $1
-  exit 1
-}
-
-ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/mariadb/archive
-RESTORE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/mariadb/restore
 ARGS=("$@")
+
+if [[ -s /tmp/restore_main.sh ]]; then
+  source /tmp/restore_main.sh
+else
+  echo "File /tmp/restore_main.sh does not exist."
+  exit 1
+fi
+
+# Export the variables needed by the framework
+export DB_NAME="mariadb"
+export DB_NAMESPACE=${MARIADB_POD_NAMESPACE}
+export ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${DB_NAMESPACE}/${DB_NAME}/archive
+
 RESTORE_USER='restoreuser'
 RESTORE_PW=$(pwgen 16 1)
 RESTORE_LOG='/tmp/restore_error.log'
 rm -f $RESTORE_LOG
 
-#Create Restore Directory
-mkdir -p $RESTORE_DIR
-
 # This is for commands which require admin access
 MYSQL="mysql \
        --defaults-file=/etc/mysql/admin_user.cnf \
@@ -42,67 +46,89 @@ RESTORE_CMD="mysql \
              --host=$MARIADB_SERVER_SERVICE_HOST \
              --connect-timeout 10"
 
-#Delete file
+# Get a single database data from the SQL file.
-delete_files() {
+# $1 - database name
-  files_to_delete=("$@")
+# $2 - sql file path
-  for f in "${files_to_delete[@]}"
+current_db_desc() {
-  do
+  PATTERN="-- Current Database:"
-    if [ -f $f ]
+  sed -n "/${PATTERN} \`$1\`/,/${PATTERN}/p" $2
-    then
-      rm -rf $f
-    fi
-  done
-}
-
-#Display all archives
-list_archives() {
-  if [ -d ${ARCHIVE_DIR} ]
-  then
-    archives=$(find ${ARCHIVE_DIR}/ -iname "*.gz" -print)
-    echo "All Archives"
-    echo "=================================="
-    for archive in $archives
-    do
-      echo $archive | cut -d '/' -f 8
-    done
-  else
-    log_error "Archive directory is not available."
-  fi
 }
 
 #Return all database from an archive
 get_databases() {
-  archive_file=$1
+  TMP_DIR=$1
-  if [ -e ${ARCHIVE_DIR}/${archive_file} ]
+  DB_FILE=$2
+
+  if [[ -e ${TMP_DIR}/db.list ]]
   then
-    files_to_purge=$(find $RESTORE_DIR/ -iname "*.sql" -print)
+    DBS=$(cat ${TMP_DIR}/db.list )
-    delete_files $files_to_purge
-    tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null
-    if [ -e ${RESTORE_DIR}/db.list ]
-    then
-      DBS=$(cat ${RESTORE_DIR}/db.list )
-    else
-      DBS=" "
-    fi
   else
     DBS=" "
   fi
+
+  echo $DBS > $DB_FILE
 }
 
-#Display all database from an archive
+# Extract all tables of a database from an archive and put them in the requested
-list_databases() {
+# file.
-  archive_file=$1
+get_tables() {
-  get_databases $archive_file
+  DATABASE=$1
-  #echo $DBS
+  TMP_DIR=$2
-  if [ -n "$DBS" ]
+  TABLE_FILE=$3
-  then
+
-    echo " "
+  SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql
-    echo "Databases in the archive $archive_file"
+  if [[ -e $TMP_DIR/$SQL_FILE ]]; then
-    echo "================================================================="
+    current_db_desc ${DATABASE} ${TMP_DIR}/${SQL_FILE} \
-    for db in $DBS
+        | grep "^CREATE TABLE" | awk -F '`' '{print $2}' \
-    do
+        > $TABLE_FILE
-      echo $db
+  else
-    done
+    # Error, cannot report the tables
+    echo "No SQL file found - cannot extract the tables"
+    return 1
+  fi
+}
+
+# Extract all rows in the given table of a database from an archive and put
+# them in the requested file.
+get_rows() {
+  DATABASE=$1
+  TABLE=$2
+  TMP_DIR=$3
+  ROW_FILE=$4
+
+  SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql
+  if [[ -e $TMP_DIR/$SQL_FILE ]]; then
+    current_db_desc ${DATABASE} ${TMP_DIR}/${SQL_FILE} \
+        | grep "INSERT INTO \`${TABLE}\` VALUES" > $ROW_FILE
+  else
+    # Error, cannot report the rows
+    echo "No SQL file found - cannot extract the rows"
+    return 1
+  fi
+}
+
+# Extract the schema for the given table in the given database belonging to
+# the archive file found in the TMP_DIR.
+get_schema() {
+  DATABASE=$1
+  TABLE=$2
+  TMP_DIR=$3
+  SCHEMA_FILE=$4
+
+  SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql
+  if [[ -e $TMP_DIR/$SQL_FILE ]]; then
+    DB_FILE=$(mktemp -p /tmp)
+    current_db_desc ${DATABASE} ${TMP_DIR}/${SQL_FILE} > ${DB_FILE}
+    sed -n /'CREATE TABLE `'$TABLE'`'/,/'--'/p ${DB_FILE} > ${SCHEMA_FILE}
+    if [[ ! (-s ${SCHEMA_FILE}) ]]; then
+      sed -n /'CREATE TABLE IF NOT EXISTS `'$TABLE'`'/,/'--'/p ${DB_FILE} \
+          > ${SCHEMA_FILE}
+    fi
+    rm -f ${DB_FILE}
+  else
+    # Error, cannot report the rows
+    echo "No SQL file found - cannot extract the schema"
+    return 1
   fi
 }
 
@@ -116,17 +142,19 @@ create_restore_user() {
   delete_restore_user "dont_exit_on_error"
 
   $MYSQL --execute="GRANT SELECT ON *.* TO ${RESTORE_USER}@'%' IDENTIFIED BY '${RESTORE_PW}';" 2>>$RESTORE_LOG
-  if [ "$?" -eq 0 ]
+  if [[ "$?" -eq 0 ]]
   then
     $MYSQL --execute="GRANT ALL ON ${restore_db}.* TO ${RESTORE_USER}@'%' IDENTIFIED BY '${RESTORE_PW}';" 2>>$RESTORE_LOG
-    if [ "$?" -ne 0 ]
+    if [[ "$?" -ne 0 ]]
     then
       cat $RESTORE_LOG
-      log_error "Failed to grant restore user ALL permissions on database ${restore_db}"
+      echo "Failed to grant restore user ALL permissions on database ${restore_db}"
+      return 1
     fi
   else
     cat $RESTORE_LOG
-    log_error "Failed to grant restore user select permissions on all databases"
+    echo "Failed to grant restore user select permissions on all databases"
+    return 1
   fi
 }
 
@@ -135,208 +163,125 @@ delete_restore_user() {
   error_handling=$1
 
   $MYSQL --execute="DROP USER ${RESTORE_USER}@'%';" 2>>$RESTORE_LOG
-  if [ "$?" -ne 0 ]
+  if [[ "$?" -ne 0 ]]
   then
     if [ "$error_handling" == "exit_on_error" ]
     then
       cat $RESTORE_LOG
-      log_error "Failed to delete temporary restore user - needs attention to avoid a security hole"
+      echo "Failed to delete temporary restore user - needs attention to avoid a security hole"
+      return 1
     fi
   fi
 }
 
 #Restore a single database
 restore_single_db() {
-  single_db_name=$1
+  SINGLE_DB_NAME=$1
-  if [ -z "$single_db_name" ]
+  TMP_DIR=$2
+
+  if [[ -z "$SINGLE_DB_NAME" ]]
   then
-    log_error "Restore single DB called but with wrong parameter."
+    echo "Restore single DB called but with wrong parameter."
+    return 1
   fi
-  if [ -f ${ARCHIVE_DIR}/${archive_file} ]
+
+  SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql
+  if [[ -f ${TMP_DIR}/$SQL_FILE ]]
   then
-    files_to_purge=$(find $RESTORE_DIR/ -iname "*.sql" -print)
+    # Restoring a single database requires us to create a temporary user
-    delete_files $files_to_purge
+    # which has capability to only restore that ONE database. One gotcha
-    tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null
+    # is that the mysql command to restore the database is going to throw
-    if [ -f ${RESTORE_DIR}/mariadb.all.sql ]
+    # errors because of all the other databases that it cannot access. So
+    # because of this reason, the --force option is used to prevent the
+    # command from stopping on an error.
+    create_restore_user $SINGLE_DB_NAME
+    if [[ $? -ne 0 ]]
     then
-      # Restoring a single database requires us to create a temporary user
+      echo "Restore $SINGLE_DB_NAME failed create restore user."
-      # which has capability to only restore that ONE database. One gotcha
+      return 1
-      # is that the mysql command to restore the database is going to throw
+    fi
-      # errors because of all the other databases that it cannot access. So
+    $RESTORE_CMD --force < ${TMP_DIR}/$SQL_FILE 2>>$RESTORE_LOG
-      # because of this reason, the --force option is used to prevent the
+    if [[ "$?" -eq 0 ]]
-      # command from stopping on an error.
+    then
-      create_restore_user $single_db_name
+      echo "Database $SINGLE_DB_NAME Restore successful."
-      $RESTORE_CMD --force < ${RESTORE_DIR}/mariadb.all.sql 2>>$RESTORE_LOG
+    else
-      if [ "$?" -eq 0 ]
+      cat $RESTORE_LOG
+      delete_restore_user "exit_on_error"
+      echo "Database $SINGLE_DB_NAME Restore failed."
+      return 1
+    fi
+    delete_restore_user "exit_on_error"
+    if [[ $? -ne 0 ]]
+    then
+      echo "Restore $SINGLE_DB_NAME failed delete restore user."
+      return 1
+    fi
+    if [ -f ${TMP_DIR}/${SINGLE_DB_NAME}_grant.sql ]
+    then
+      $MYSQL < ${TMP_DIR}/${SINGLE_DB_NAME}_grant.sql 2>>$RESTORE_LOG
+      if [[ "$?" -eq 0 ]]
       then
-        echo "Database $single_db_name Restore successful."
+        echo "Database $SINGLE_DB_NAME Permission Restore successful."
       else
         cat $RESTORE_LOG
-        delete_restore_user "exit_on_error"
+        echo "Database $SINGLE_DB_NAME Permission Restore failed."
-        log_error "Database $single_db_name Restore failed."
+        return 1
-      fi
-      delete_restore_user "exit_on_error"
-
-      if [ -f ${RESTORE_DIR}/${single_db_name}_grant.sql ]
-      then
-        $MYSQL < ${RESTORE_DIR}/${single_db_name}_grant.sql 2>>$RESTORE_LOG
-        if [ "$?" -eq 0 ]
-        then
-          echo "Database $single_db_name Permission Restore successful."
-        else
-          cat $RESTORE_LOG
-          log_error "Database $single_db_name Permission Restore failed."
-        fi
-      else
-        log_error "There is no permission file available for $single_db_name"
       fi
     else
-      log_error "There is no database file available to restore from"
+      echo "There is no permission file available for $SINGLE_DB_NAME"
+      return 1
     fi
   else
-    log_error "Archive does not exist"
+    echo "There is no database file available to restore from"
+    return 1
   fi
+  return 0
 }
 
 #Restore all the databases
 restore_all_dbs() {
-  if [ -f ${ARCHIVE_DIR}/${archive_file} ]
+  TMP_DIR=$1
+
+  SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql
+  if [[ -f ${TMP_DIR}/$SQL_FILE ]]
   then
-    files_to_purge=$(find $RESTORE_DIR/ -iname "*.sql" -print)
+    $MYSQL < ${TMP_DIR}/$SQL_FILE 2>$RESTORE_LOG
-    delete_files $files_to_purge
+    if [[ "$?" -eq 0 ]]
-    tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null
-    if [ -f ${RESTORE_DIR}/mariadb.all.sql ]
     then
-      $MYSQL < ${RESTORE_DIR}/mariadb.all.sql 2>$RESTORE_LOG
+      echo "Databases $( echo $DBS | tr -d '\n') Restore successful."
-      if [ "$?" -eq 0 ]
-      then
-        echo "Databases $( echo $DBS | tr -d '\n') Restore successful."
-      else
-        cat $RESTORE_LOG
-        log_error "Databases $( echo $DBS | tr -d '\n') Restore failed."
-      fi
-      if [ -n "$DBS" ]
-      then
-        for db in $DBS
-        do
-          if [ -f ${RESTORE_DIR}/${db}_grant.sql ]
-          then
-            $MYSQL < ${RESTORE_DIR}/${db}_grant.sql 2>>$RESTORE_LOG
-            if [ "$?" -eq 0 ]
-            then
-              echo "Database $db Permission Restore successful."
-            else
-              cat $RESTORE_LOG
-              log_error "Database $db Permission Restore failed."
-            fi
-          else
-            log_error "There is no permission file available for $db"
-          fi
-        done
     else
-      log_error "There is no database file available to restore from"
+      cat $RESTORE_LOG
+      echo "Databases $( echo $DBS | tr -d '\n') Restore failed."
+      return 1
     fi
-  else
+    if [ -n "$DBS" ]
-    log_error "Archive does not exist"
-  fi
- fi
-}
-
-usage() {
-  ret_val=$1
-  echo "Usage:"
-  echo "Restore command options"
-  echo "============================="
-  echo "help"
-  echo "list_archives"
-  echo "list_databases <archive_filename>"
-  echo "restore <archive_filename> [<db_name> | ALL]"
-  exit $ret_val
-}
-
-is_Option() {
-  opts=$1
-  param=$2
-  find=0
-  for opt in $opts
-  do
-    if [ "$opt" == "$param" ]
     then
-      find=1
+      for db in $DBS
-    fi
+      do
-  done
+        if [ -f ${TMP_DIR}/${db}_grant.sql ]
-  echo $find
-}
-
-#Main
-if [ ${#ARGS[@]} -gt 3 ]
-then
-  usage 1
-elif [ ${#ARGS[@]} -eq 1 ]
-then
-  if [ "${ARGS[0]}" == "list_archives" ]
-  then
-    list_archives
-  elif [ "${ARGS[0]}" == "help" ]
-  then
-    usage 0
-  else
-    usage 1
-  fi
-elif [ ${#ARGS[@]} -eq 2 ]
-then
-  if [ "${ARGS[0]}" == "list_databases" ]
-  then
-    list_databases ${ARGS[1]}
-  else
-    usage 1
-  fi
-elif [ ${#ARGS[@]} -eq 3 ]
-then
-  if [ "${ARGS[0]}" != "restore" ]
-  then
-    usage 1
-  else
-    if [ -f ${ARCHIVE_DIR}/${ARGS[1]} ]
-    then
-      #Get all the databases in that archive
-      get_databases ${ARGS[1]}
-
-      #check if the requested database is available in the archive
-      if [ $(is_Option "$DBS" ${ARGS[2]}) -eq 1 ]
-      then
-        echo "Creating database ${ARGS[2]} if it does not exist"
-        $MYSQL -e "CREATE DATABASE IF NOT EXISTS \`${ARGS[2]}\`" 2>>$RESTORE_LOG
-        if [ "$?" -ne 0 ]
         then
-          cat $RESTORE_LOG
+          $MYSQL < ${TMP_DIR}/${db}_grant.sql 2>>$RESTORE_LOG
-          log_error "Database ${ARGS[2]} could not be created."
+          if [[ "$?" -eq 0 ]]
-        fi
-        echo "Restoring database ${ARGS[2]} and grants...this could take a few minutes."
-        restore_single_db ${ARGS[2]}
-      elif [ "$( echo ${ARGS[2]} | tr '[a-z]' '[A-Z]')" == "ALL" ]
-      then
-        echo "Creating databases if they do not exist"
-        for db in $DBS
-        do
-          $MYSQL -e "CREATE DATABASE IF NOT EXISTS \`$db\`"
-          if [ "$?" -ne 0 ]
           then
+            echo "Database $db Permission Restore successful."
+          else
             cat $RESTORE_LOG
-            log_error "Database ${db} could not be created."
+            echo "Database $db Permission Restore failed."
+            return 1
           fi
-        done
+        else
-        echo "Restoring all databases and grants...this could take a few minutes."
+          echo "There is no permission file available for $db"
-        restore_all_dbs
+          return 1
-      else
+        fi
-        echo "Database ${ARGS[2]} does not exist."
+      done
-      fi
-    else
-      echo "Archive file not found"
     fi
+  else
+    echo "There is no database file available to restore from"
+    return 1
   fi
-else
+  return 0
-  usage 1
+}
-fi
 
-exit 0
+# Call the CLI interpreter, providing the archive directory path and the
+# user arguments passed in
+cli_main ${ARGS[@]}

mariadb/templates/configmap-bin.yaml

@@ -42,5 +42,13 @@ data:
 {{ tuple "bin/_backup_mariadb.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   restore_mariadb.sh: |
 {{ tuple "bin/_restore_mariadb.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  backup_main.sh: |
+{{ include "helm-toolkit.scripts.db-backup-restore.backup_main" . | indent 4 }}
+  restore_main.sh: |
+{{ include "helm-toolkit.scripts.db-backup-restore.restore_main" . | indent 4 }}
+{{- end }}
+{{- if .Values.manifests.job_ks_user }}
+  ks-user.sh: |
+{{ include "helm-toolkit.scripts.keystone_user" . | indent 4 }}
 {{- end }}
 {{- end }}

mariadb/templates/cron-job-backup-mariadb.yaml

@@ -27,6 +27,12 @@ metadata:
   labels:
 {{ tuple $envAll "mariadb-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
 spec:
+{{- if .Values.jobs.backup_mariadb.backoffLimit }}
+  backoffLimit: {{ .Values.jobs.backup_mariadb.backoffLimit }}
+{{- end }}
+{{- if .Values.jobs.backup_mariadb.activeDeadlineSeconds }}
+  activeDeadlineSeconds: {{ .Values.jobs.backup_mariadb.activeDeadlineSeconds }}
+{{- end }}
   schedule: {{ .Values.jobs.backup_mariadb.cron | quote }}
   successfulJobsHistoryLimit: {{ .Values.jobs.backup_mariadb.history.success }}
   failedJobsHistoryLimit: {{ .Values.jobs.backup_mariadb.history.failed }}
@@ -36,7 +42,7 @@ spec:
       labels:
 {{ tuple $envAll "mariadb-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
       annotations:
-{{ dict "envAll" $envAll "podName" "mariadb-backup" "containerNames" (list "init" "mariadb-backup") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
+{{ dict "envAll" $envAll "podName" "mariadb-backup" "containerNames" (list "init" "backup-perms" "mariadb-backup") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
     spec:
       template:
         metadata:
@@ -48,7 +54,24 @@ spec:
           nodeSelector:
             {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
           initContainers:
-{{ tuple $envAll "mariadb_backup" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 10 }}
+{{ tuple $envAll "mariadb_backup" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 12 }}
+            - name: backup-perms
+{{ tuple $envAll "mariadb_backup" | include "helm-toolkit.snippets.image" | indent 14 }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }}
+{{ dict "envAll" $envAll "application" "mariadb_backup" "container" "backup_perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }}
+              command:
+                - chown
+                - -R
+                - "65534:65534"
+                - $(MARIADB_BACKUP_BASE_DIR)
+              env:
+                - name: MARIADB_BACKUP_BASE_DIR
+                  value: {{ .Values.conf.backup.base_path | quote }}
+              volumeMounts:
+                - mountPath: /tmp
+                  name: pod-tmp
+                - mountPath: {{ .Values.conf.backup.base_path }}
+                  name: mariadb-backup-dir
           containers:
             - name: mariadb-backup
               command:
@@ -58,14 +81,28 @@ spec:
                   value: {{ .Values.conf.backup.base_path | quote }}
                 - name: MYSQL_BACKUP_MYSQLDUMP_OPTIONS
                   value: {{ .Values.conf.backup.mysqldump_options | quote }}
-                - name: MARIADB_BACKUP_DAYS_TO_KEEP
+                - name: MARIADB_LOCAL_BACKUP_DAYS_TO_KEEP
-                  value: {{ .Values.conf.backup.days_of_backup_to_keep | quote }}
+                  value: {{ .Values.conf.backup.days_to_keep | quote }}
                 - name: MARIADB_POD_NAMESPACE
                   valueFrom:
                     fieldRef:
                       fieldPath: metadata.namespace
+                - name: REMOTE_BACKUP_ENABLED
+                  value: "{{ .Values.conf.backup.remote_backup.enabled }}"
+{{- if .Values.conf.backup.remote_backup.enabled }}
+                - name: MARIADB_REMOTE_BACKUP_DAYS_TO_KEEP
+                  value: {{ .Values.conf.backup.remote_backup.days_to_keep | quote }}
+                - name: CONTAINER_NAME
+                  value: {{ .Values.conf.backup.remote_backup.container_name | quote }}
+                - name: STORAGE_POLICY
+                  value: "{{ .Values.conf.backup.remote_backup.storage_policy }}"
+{{- with $env := dict "ksUserSecret" $envAll.Values.secrets.identity.remote_rgw_user }}
+{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 16 }}
+{{- end }}
+{{- end }}
 {{ tuple $envAll "mariadb_backup" | include "helm-toolkit.snippets.image" | indent 14 }}
 {{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }}
+{{ dict "envAll" $envAll "application" "mariadb_backup" "container" "mariadb_backup" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }}
               volumeMounts:
                 - name: pod-tmp
                   mountPath: /tmp
@@ -73,6 +110,10 @@ spec:
                   name: mariadb-bin
                   readOnly: true
                   subPath: backup_mariadb.sh
+                - mountPath: /tmp/backup_main.sh
+                  name: mariadb-bin
+                  readOnly: true
+                  subPath: backup_main.sh
                 - mountPath: {{ .Values.conf.backup.base_path }}
                   name: mariadb-backup-dir
                 - name: mariadb-secrets
@@ -88,7 +129,7 @@ spec:
             - name: mariadb-secrets
               secret:
                 secretName: mariadb-secrets
-                defaultMode: 384
+                defaultMode: 420
             - configMap:
                 defaultMode: 365
                 name: mariadb-bin

mariadb/templates/job-ks-user.yaml

@@ -0,0 +1,20 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.job_ks_user }}
+{{- $backoffLimit := .Values.jobs.ks_user.backoffLimit }}
+{{- $activeDeadlineSeconds := .Values.jobs.ks_user.activeDeadlineSeconds }}
+{{- $ksUserJob := dict "envAll" . "serviceName" "mariadb" "configMapBin" "mariadb-bin" "backoffLimit" $backoffLimit "activeDeadlineSeconds" $activeDeadlineSeconds -}}
+{{ $ksUserJob | include "helm-toolkit.manifests.job_ks_user" }}
+{{- end }}

mariadb/templates/secret-backup-restore.yaml

@@ -0,0 +1,27 @@
+{{/*
+This manifest results a secret being created which has the key information
+needed for backing up and restoring the Mariadb databases.
+*/}}
+
+{{- if and .Values.conf.backup.enabled .Values.manifests.secret_backup_restore }}
+
+{{- $envAll := . }}
+{{- $userClass := "backup_restore" }}
+{{- $secretName := index $envAll.Values.secrets.mariadb $userClass }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $secretName }}
+type: Opaque
+data:
+  BACKUP_ENABLED: {{ $envAll.Values.conf.backup.enabled | quote | b64enc }}
+  BACKUP_BASE_PATH: {{ $envAll.Values.conf.backup.base_path | b64enc }}
+  LOCAL_DAYS_TO_KEEP: {{ $envAll.Values.conf.backup.days_to_keep | quote | b64enc }}
+  MYSQLDUMP_OPTIONS: {{ $envAll.Values.conf.backup.mysqldump_options | b64enc }}
+  REMOTE_BACKUP_ENABLED: {{ $envAll.Values.conf.backup.remote_backup.enabled | quote | b64enc }}
+  REMOTE_BACKUP_CONTAINER: {{ $envAll.Values.conf.backup.remote_backup.container_name | b64enc }}
+  REMOTE_BACKUP_DAYS_TO_KEEP: {{ $envAll.Values.conf.backup.remote_backup.days_to_keep | quote | b64enc }}
+  REMOTE_BACKUP_STORAGE_POLICY: {{ $envAll.Values.conf.backup.remote_backup.storage_policy | b64enc }}
+...
+{{- end }}

mariadb/templates/secret-rgw.yaml

@@ -0,0 +1,78 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+This manifest results in two secrets being created:
+  1) Keystone "remote_rgw_user" secret, which is needed to access the cluster
+     (remote or same cluster) for storing mariadb backups. If the
+     cluster is remote, the auth_url would be non-null.
+  2) Keystone "remote_ks_admin" secret, which is needed to create the
+     "remote_rgw_user" keystone account mentioned above. This may not
+     be needed if the account is in a remote cluster (auth_url is non-null
+     in that case).
+*/}}
+
+{{- if .Values.conf.backup.remote_backup.enabled }}
+
+{{- $envAll := . }}
+{{- $userClass := "remote_rgw_user" }}
+{{- $secretName := index $envAll.Values.secrets.identity $userClass }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $secretName }}
+type: Opaque
+data:
+{{- $identityClass := index .Values.endpoints.identity.auth $userClass }}
+{{- if $identityClass.auth_url }}
+  OS_AUTH_URL: {{ $identityClass.auth_url | b64enc }}
+{{- else }}
+  OS_AUTH_URL: {{ tuple "identity" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | b64enc }}
+{{- end }}
+  OS_REGION_NAME: {{ $identityClass.region_name | b64enc }}
+  OS_INTERFACE: {{ $identityClass.interface | default "internal" | b64enc }}
+  OS_PROJECT_DOMAIN_NAME: {{ $identityClass.project_domain_name | b64enc }}
+  OS_PROJECT_NAME: {{ $identityClass.project_name | b64enc }}
+  OS_USER_DOMAIN_NAME: {{ $identityClass.user_domain_name | b64enc }}
+  OS_USERNAME: {{ $identityClass.username | b64enc }}
+  OS_PASSWORD: {{ $identityClass.password | b64enc }}
+  OS_DEFAULT_DOMAIN: {{ $identityClass.default_domain_id | default "default" | b64enc }}
+...
+{{- if .Values.manifests.job_ks_user }}
+{{- $userClass := "remote_ks_admin" }}
+{{- $secretName := index $envAll.Values.secrets.identity $userClass }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $secretName }}
+type: Opaque
+data:
+{{- $identityClass := index .Values.endpoints.identity.auth $userClass }}
+{{- if $identityClass.auth_url }}
+  OS_AUTH_URL: {{ $identityClass.auth_url | b64enc }}
+{{- else }}
+  OS_AUTH_URL: {{ tuple "identity" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | b64enc }}
+{{- end }}
+  OS_REGION_NAME: {{ $identityClass.region_name | b64enc }}
+  OS_INTERFACE: {{ $identityClass.interface | default "internal" | b64enc }}
+  OS_PROJECT_DOMAIN_NAME: {{ $identityClass.project_domain_name | b64enc }}
+  OS_PROJECT_NAME: {{ $identityClass.project_name | b64enc }}
+  OS_USER_DOMAIN_NAME: {{ $identityClass.user_domain_name | b64enc }}
+  OS_USERNAME: {{ $identityClass.username | b64enc }}
+  OS_PASSWORD: {{ $identityClass.password | b64enc }}
+  OS_DEFAULT_DOMAIN: {{ $identityClass.default_domain_id | default "default" | b64enc }}
+...
+{{- end }}
+{{- end }}

mariadb/values.yaml

@@ -29,7 +29,8 @@ images:
     prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:newton-ubuntu_xenial
     dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
     image_repo_sync: docker.io/docker:17.07.0
-    mariadb_backup: docker.io/openstackhelm/mariadb:ubuntu_xenial-20191031
+    mariadb_backup: quay.io/airshipit/porthole-mysqlclient-utility:latest-ubuntu_bionic
+    ks_user: docker.io/openstackhelm/heat:stein-ubuntu_bionic
     scripted_test: docker.io/openstackhelm/mariadb:ubuntu_xenial-20191031
   pull_policy: "IfNotPresent"
   local_registry:
@@ -109,6 +110,17 @@ pod:
         main:
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true
+    mariadb_backup:
+      pod:
+        runAsUser: 65534
+      container:
+        backup_perms:
+          runAsUser: 0
+          readOnlyRootFilesystem: true
+        mariadb_backup:
+          runAsUser: 65534
+          readOnlyRootFilesystem: true
+          allowPrivilegeEscalation: false
     tests:
       pod:
         runAsUser: 999
@@ -190,6 +202,13 @@ pod:
         limits:
           memory: "1024Mi"
           cpu: "2000m"
+      ks_user:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
 
 dependencies:
   dynamic:
@@ -208,8 +227,9 @@ dependencies:
       services:
         - endpoint: error_pages
           service: oslo_db
-    mariadb:
+    backup_mariadb:
-      jobs: null
+      jobs:
+        - mariadb-ks-user
       services: null
     prometheus_create_mysql_user:
       services:
@@ -260,10 +280,17 @@ jobs:
     backoffLimit: 87600
     activeDeadlineSeconds: 3600
   backup_mariadb:
+    # activeDeadlineSeconds == 0 means no deadline
+    activeDeadlineSeconds: 0
+    backoffLimit: 6
     cron: "0 0 * * *"
     history:
       success: 3
       failed: 1
+  ks_user:
+    # activeDeadlineSeconds == 0 means no deadline
+    activeDeadlineSeconds: 0
+    backoffLimit: 6
 
 conf:
   tests:
@@ -284,12 +311,17 @@ conf:
   ingress_conf:
     worker-processes: "auto"
   backup:
-    enabled: true
+    enabled: false
     base_path: /var/backup
     mysqldump_options: >
       --single-transaction --quick --add-drop-database
       --add-drop-table --add-locks --databases
-    days_of_backup_to_keep: 3
+    days_to_keep: 3
+    remote_backup:
+      enabled: false
+      container_name: mariadb
+      days_to_keep: 14
+      storage_policy: default-placement
   database:
     my: |
       [mysqld]
@@ -407,6 +439,13 @@ monitoring:
     mysqld_exporter:
       scrape: true
 
+secrets:
+  identity:
+    remote_ks_admin: keystone-admin-user
+    remote_rgw_user: mariadb-backup-user
+  mariadb:
+    backup_restore: mariadb-backup-restore
+
 # typically overridden by environmental
 # values, but should include all endpoints
 # required by this chart
@@ -498,6 +537,44 @@ endpoints:
       dns:
         default: 53
         protocol: UDP
+  identity:
+    name: backup-storage-auth
+    namespace: openstack
+    auth:
+      remote_ks_admin:
+        # Auth URL of null indicates local authentication
+        # HTK will form the URL unless specified here
+        auth_url: null
+        region_name: RegionOne
+        username: admin
+        password: password
+        project_name: admin
+        user_domain_name: default
+        project_domain_name: default
+      remote_rgw_user:
+        # Auth URL of null indicates local authentication
+        # HTK will form the URL unless specified here
+        auth_url: null
+        role: admin
+        region_name: RegionOne
+        username: mariadb-backup-user
+        password: password
+        project_name: service
+        user_domain_name: service
+        project_domain_name: service
+    hosts:
+      default: keystone
+      internal: keystone-api
+    host_fqdn_override:
+      default: null
+    path:
+      default: /v3
+    scheme:
+      default: 'http'
+    port:
+      api:
+        default: 80
+        internal: 5000
 
 network_policy:
   mariadb:
@@ -521,6 +598,7 @@ manifests:
   deployment_ingress: true
   job_image_repo_sync: true
   cron_job_mariadb_backup: false
+  job_ks_user: false
   pvc_backup: false
   monitoring:
     prometheus:
@@ -536,6 +614,7 @@ manifests:
   secret_dbadmin_password: true
   secret_sst_password: true
   secret_dbaudit_password: true
+  secret_backup_restore: false
   secret_etc: true
   service_discovery: true
   service_ingress: true