Add mariadb backup capability to mariadb chart

* backup script for mariadb
 * restore script for mariadb
 * cronjob to control backup automation for mariadb
 * add parameters to values.yaml

Change-Id: Ib0b6744f519ea1919507df0c2a50b45b27ac86ab
This commit is contained in:
Koffi Nogbe 2019-03-20 10:34:24 -04:00 committed by Tin Lam
parent 8673bdda53
commit 2379b94b3d
6 changed files with 573 additions and 5 deletions

View File

@ -0,0 +1,132 @@
#!/bin/bash
# Copyright 2018 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -x
BACKUPS_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/mariadb/current
ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/mariadb/archive
MYSQL="mysql \
--defaults-file=/etc/mysql/admin_user.cnf \
--host=$MARIADB_SERVER_SERVICE_HOST \
--connect-timeout 10"
MYSQLDUMP="mysqldump \
--defaults-file=/etc/mysql/admin_user.cnf \
--host=$MARIADB_SERVER_SERVICE_HOST"
delete_files() {
files_to_delete=("$@")
for f in "${files_to_delete[@]}"
do
if [ -f $f ]
then
echo "Deleting file $f."
rm -rf $f
fi
done
}
days_difference() {
archive_date=$( date --date="$1" +%s )
if [ "$?" -ne 0 ]
then
day_delta=0
fi
current_date=$( date +%s )
date_delta=$(($current_date-$archive_date))
if [ "$date_delta" -lt 0 ]
then
day_delta=0
else
day_delta=$(($date_delta/86400))
fi
echo $day_delta
}
DBNAME=( $($MYSQL --silent --skip-column-names -e \
"show databases;" | \
egrep -vi 'information_schema|performance_schema|mysql') )
#check if there is a database to backup, otherwise exit
if [[ -z "${DBNAME// }" ]]
then
echo "There is no database to backup"
exit 0
fi
#Create archive and backup directories.
mkdir -p $BACKUPS_DIR $ARCHIVE_DIR
#Create a list of Databases
printf "%s\n" "${DBNAME[@]}" > $BACKUPS_DIR/db.list
#Retrieve and create the GRANT files per DB
for db in "${DBNAME[@]}"
do
echo $($MYSQL --skip-column-names -e "select concat('show grants for ',user,';') \
from mysql.db where ucase(db)=ucase('$db');") | \
$MYSQL --silent --skip-column-names 2>grant_err.log > $BACKUPS_DIR/${db}_grant.sql
if [ "$?" -eq 0 ]
then
sed -i 's/$/;/' $BACKUPS_DIR/${db}_grant.sql
else
cat grant_err.log
fi
done
#Dumping the database
#DATE=$(date +"%Y_%m_%d_%H_%M_%S")
DATE=$(date +'%Y-%m-%dT%H:%M:%SZ')
$MYSQLDUMP $MYSQL_BACKUP_MYSQLDUMP_OPTIONS "${DBNAME[@]}" \
> $BACKUPS_DIR/mariadb.all.sql 2>dberror.log
if [[ $? -eq 0 && -s $BACKUPS_DIR/mariadb.all.sql ]]
then
#Archive the current db files
pushd $BACKUPS_DIR 1>/dev/null
tar zcvf $ARCHIVE_DIR/mariadb.all.${DATE}.tar.gz *
ARCHIVE_RET=$?
popd 1>/dev/null
else
#TODO: This can be convert into mail alert of alert send to a monitoring system
echo "Backup failed and need attention."
cat dberror.log
exit 1
fi
#Remove the current backup
if [ -d $BACKUPS_DIR ]
then
rm -rf $BACKUPS_DIR/*.sql
fi
#Only delete the old archive after a successful archive
if [ $ARCHIVE_RET -eq 0 ]
then
if [ "$MARIADB_BACKUP_DAYS_TO_KEEP" -gt 0 ]
then
echo "Deleting backups older than $MARIADB_BACKUP_DAYS_TO_KEEP days"
if [ -d $ARCHIVE_DIR ]
then
for archive_file in $(ls -1 $ARCHIVE_DIR/*.gz)
do
archive_date=$( echo $archive_file | awk -F/ '{print $NF}' | cut -d'.' -f 3)
if [ "$(days_difference $archive_date)" -gt "$MARIADB_BACKUP_DAYS_TO_KEEP" ]
then
rm -rf $archive_file
fi
done
fi
fi
fi

View File

@ -0,0 +1,269 @@
#!/bin/bash
# Copyright 2018 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/mariadb/archive
RESTORE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/mariadb/restore
ARGS=("$@")
LIST_OPTIONS=(list_archives list_databases)
#Create Restore Directory
mkdir -p $RESTORE_DIR
MYSQL="mysql \
--defaults-file=/etc/mysql/admin_user.cnf \
--host=$MARIADB_SERVER_SERVICE_HOST \
--connect-timeout 10"
#Delete file
delete_files() {
files_to_delete=("$@")
for f in "${files_to_delete[@]}"
do
if [ -f $f ]
then
rm -rf $f
fi
done
}
#Display all archives
list_archives() {
if [ -d ${ARCHIVE_DIR} ]
then
archives=$(find ${ARCHIVE_DIR}/ -iname "*.gz" -print)
echo "All Archives"
echo "=================================="
for archive in $archives
do
echo $archive | cut -d '/' -f 8
done
exit 0
else
echo "Archive directory is not available."
exit 1
fi
}
#Return all database from an archive
get_databases() {
archive_file=$1
if [ -e ${ARCHIVE_DIR}/${archive_file} ]
then
files_to_purge=$(find $RESTORE_DIR/ -iname "*.sql" -print)
delete_files $files_to_purge
tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null
if [ -e ${RESTORE_DIR}/db.list ]
then
DBS=$(cat ${RESTORE_DIR}/db.list )
else
DBS=" "
fi
else
DBS=" "
fi
}
#Display all database from an archive
list_databases() {
archive_file=$1
get_databases $archive_file
#echo $DBS
if [ -n "$DBS" ]
then
echo " "
echo "Databases in the archive $archive_file"
echo "================================================================="
for db in $DBS
do
echo $db
done
fi
}
#Restore a single database
restore_single_db() {
single_db_name=$1
if [ -z "$single_db_name" ]
then
usage
exit 1
fi
if [ -f ${ARCHIVE_DIR}/${archive_file} ]
then
files_to_purge=$(find $RESTORE_DIR/ -iname "*.sql" -print)
delete_files $files_to_purge
tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null
if [ -f ${RESTORE_DIR}/mariadb.all.sql ]
then
$MYSQL --one-database $single_db_name < ${RESTORE_DIR}/mariadb.all.sql
if [ "$?" -eq 0 ]
then
echo "Database $single_db_name Restore successful."
else
echo "Database $single_db_name Restore failed."
fi
if [ -f ${RESTORE_DIR}/${single_db_name}_grant.sql ]
then
$MYSQL < ${RESTORE_DIR}/${single_db_name}_grant.sql
if [ "$?" -eq 0 ]
then
echo "Database $single_db_name Permission Restore successful."
else
echo "Database $single_db_name Permission Restore failed."
fi
else
echo "There is no permission file available for $single_db_name"
fi
else
echo "There is no database file available to restore from"
fi
else
echo "Archive does not exist"
fi
}
#Restore all the databases
restore_all_dbs() {
if [ -f ${ARCHIVE_DIR}/${archive_file} ]
then
files_to_purge=$(find $RESTORE_DIR/ -iname "*.sql" -print)
delete_files $files_to_purge
tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null
if [ -f ${RESTORE_DIR}/mariadb.all.sql ]
then
$MYSQL < ${RESTORE_DIR}/mariadb.all.sql
if [ "$?" -eq 0 ]
then
echo "Databases $( echo $DBS | tr -d '\n') Restore successful."
else
echo "Databases $( echo $DBS | tr -d '\n') Restore failed."
fi
if [ -n "$DBS" ]
then
for db in $DBS
do
if [ -f ${RESTORE_DIR}/${db}_grant.sql ]
then
$MYSQL < ${RESTORE_DIR}/${db}_grant.sql
if [ "$?" -eq 0 ]
then
echo "Database $db Permission Restore successful."
else
echo "Database $db Permission Restore failed."
fi
else
echo "There is no permission file available for $db"
fi
done
else
echo "There is no database file available to restore from"
fi
else
echo "Archive does not exist"
fi
fi
}
usage() {
echo "Usage:"
echo "$0 options"
echo "============================="
echo "options: "
echo "list_archives"
echo "list_databases archive_filename"
echo "restore archive_filename [DB_NAME or ALL/all]"
}
is_Option() {
opts=$1
param=$2
find=0
for opt in $opts
do
if [ "$opt" == "$param" ]
then
find=1
fi
done
echo $find
}
#Main
if [ ${#ARGS[@]} -gt 3 ]
then
usage
exit
elif [ ${#ARGS[@]} -eq 1 ]
then
if [ $(is_Option "$LIST_OPTIONS" ${ARGS[0]}) -eq 1 ]
then
${ARGS[0]}
exit
else
usage
exit
fi
elif [ ${#ARGS[@]} -eq 2 ]
then
if [ "${ARGS[0]}" == "list_databases" ]
then
list_databases ${ARGS[1]}
exit 0
else
usage
exit
fi
elif [ ${#ARGS[@]} -eq 3 ]
then
if [ "${ARGS[0]}" != "restore" ]
then
usage
exit 1
else
if [ -f ${ARCHIVE_DIR}/${ARGS[1]} ]
then
#Get all the databases in that archive
get_databases ${ARGS[1]}
#check if the requested database is available in the archive
if [ $(is_Option "$DBS" ${ARGS[2]}) -eq 1 ]
then
echo "Restoring Database ${ARGS[2]} And Grants"
echo "Creating Database ${ARGS[2]} if it does not exist"
$MYSQL -e "CREATE DATABASE IF NOT EXISTS \`${ARGS[2]}\`"
restore_single_db ${ARGS[2]}
exit 0
elif [ "$( echo ${ARGS[2]} | tr '[a-z]' '[A-Z]')" == "ALL" ]
then
echo "Restoring All The Database."
echo "Creating Database if it does not exist"
for db in $DBS
do
$MYSQL -e "CREATE DATABASE IF NOT EXISTS \`$db\`"
done
restore_all_dbs
exit 0
else
echo "Database ${ARGS[2]} does not exist."
fi
else
echo "Archive file not found"
fi
fi
else
usage
exit
fi

View File

@ -36,4 +36,10 @@ data:
{{ tuple "bin/_start.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
stop.sh: |
{{ tuple "bin/_stop.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- if .Values.conf.backup.enabled }}
backup_mariadb.sh: |
{{ tuple "bin/_backup_mariadb.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
restore_mariadb.sh: |
{{ tuple "bin/_restore_mariadb.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- end }}
{{- end }}

View File

@ -0,0 +1,98 @@
{{/*
Copyright 2017 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.cron_job_mariadb_backup }}
{{- $envAll := . }}
{{- $serviceAccountName := "mariadb-backup" }}
{{ tuple $envAll "mariadb_account" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
---
apiVersion: batch/v1beta1
kind: CronJob
metadata:
name: mariadb-backup
annotations:
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
spec:
schedule: {{ .Values.jobs.backup_mariadb.cron | quote }}
successfulJobsHistoryLimit: {{ .Values.jobs.backup_mariadb.history.success }}
failedJobsHistoryLimit: {{ .Values.jobs.backup_mariadb.history.failed }}
concurrencyPolicy: Forbid
jobTemplate:
metadata:
labels:
{{ tuple $envAll "mariadb-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
spec:
template:
metadata:
labels:
{{ tuple $envAll "mariadb-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 12 }}
spec:
serviceAccountName: {{ $serviceAccountName }}
restartPolicy: OnFailure
nodeSelector:
{{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
containers:
- command:
- /tmp/backup_mariadb.sh
env:
- name: MARIADB_BACKUP_BASE_DIR
value: {{ .Values.conf.backup.base_path }}
- name: MYSQL_BACKUP_MYSQLDUMP_OPTIONS
value: {{ .Values.conf.backup.mysqldump_options }}
- name: MARIADB_BACKUP_DAYS_TO_KEEP
value: "{{ .Values.conf.backup.days_of_backup_to_keep }}"
- name: MARIADB_POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
{{ tuple $envAll "mariadb_backup" | include "helm-toolkit.snippets.image" | indent 12 }}
{{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 12 }}
name: mariadb-backup
volumeMounts:
- mountPath: /tmp/backup_mariadb.sh
name: mariadb-bin
readOnly: true
subPath: backup_mariadb.sh
- mountPath: {{ .Values.conf.backup.base_path }}
name: mariadb-backup-dir
- name: mariadb-secrets
mountPath: /etc/mysql/admin_user.cnf
subPath: admin_user.cnf
readOnly: true
restartPolicy: OnFailure
serviceAccount: {{ $serviceAccountName }}
serviceAccountName: {{ $serviceAccountName }}
volumes:
- name: mariadb-secrets
secret:
secretName: mariadb-secrets
defaultMode: 384
- configMap:
defaultMode: 365
name: mariadb-bin
name: mariadb-bin
{{- if and .Values.volume.backup.enabled .Values.manifests.pvc_backup }}
- name: mariadb-backup-dir
persistentVolumeClaim:
claimName: mariadb-backup-data
{{- else }}
- hostPath:
path: {{ .Values.conf.backup.base_path }}
type: DirectoryOrCreate
name: mariadb-backup-dir
{{- end }}
{{- end }}

View File

@ -0,0 +1,30 @@
{{/*
Copyright 2017 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if and .Values.volume.backup.enabled .Values.manifests.pvc_backup }}
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: mariadb-backup-data
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: {{ .Values.volume.backup.size }}
storageClassName: {{ .Values.volume.backup.class_name }}
{{- end }}

View File

@ -29,6 +29,7 @@ images:
prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:newton
dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
image_repo_sync: docker.io/docker:17.07.0
mariadb_backup: docker.io/openstackhelm/mariadb:10.2.18
pull_policy: "IfNotPresent"
local_registry:
active: false
@ -49,6 +50,9 @@ labels:
error_server:
node_selector_key: openstack-control-plane
node_selector_value: enabled
job:
node_selector_key: openstack-control-plane
node_selector_value: enabled
pod:
user:
@ -119,11 +123,13 @@ pod:
limits:
memory: "1024Mi"
cpu: "2000m"
jobs:
exporter_create_sql_user:
backoffLimit: 87600
activeDeadlineSeconds: 3600
mariadb_backup:
requests:
memory: "128Mi"
cpu: "100m"
limits:
memory: "1024Mi"
cpu: "2000m"
dependencies:
dynamic:
@ -165,6 +171,10 @@ dependencies:
services:
- endpoint: internal
service: local_image_registry
mariadb_backup:
services:
- endpoint: internal
service: oslo_db
force_bootstrap: false
@ -173,9 +183,30 @@ volume:
enabled: true
class_name: general
size: 5Gi
backup:
enabled: true
class_name: general
size: 5Gi
jobs:
exporter_create_sql_user:
backoffLimit: 87600
activeDeadlineSeconds: 3600
backup_mariadb:
cron: "0 0 * * *"
history:
success: 3
failed: 1
conf:
ingress: null
backup:
enabled: true
base_path: /var/backup
mysqldump_options: >
--single-transaction --quick --skip-opt
--add-drop-database --add-drop-table --databases
days_of_backup_to_keep: 3
database:
config_override: null
@ -292,6 +323,8 @@ manifests:
deployment_error: true
deployment_ingress: true
job_image_repo_sync: true
cron_job_mariadb_backup: false
pvc_backup: false
monitoring:
prometheus:
configmap_bin: true