dda97505d8
I spent a number of hours recently trying to debug why mysqldump stopped working. In reality it appears that the problem was borg-backup's ssh connectivity causing the mysql-dump to not be able to stream properly. Add a logging note about checking the other half of backup streaming when the streaming script fails to try and make it more obvious this may ocurr. Change-Id: Idf53fc9e61f05077b954d730d88beab0cc1db09b
73 lines
2.3 KiB
Django/Jinja
73 lines
2.3 KiB
Django/Jinja
#!/bin/bash
|
|
|
|
# Flags based on
|
|
# https://borgbackup.readthedocs.io/en/stable/quickstart.html
|
|
|
|
if [ -z "$1" ]; then
|
|
echo "Must specify backup host"
|
|
exit 1
|
|
fi
|
|
|
|
BORG="/opt/borg/bin/borg"
|
|
BORG_CREATE="${BORG} create --verbose --filter AME --list --stats --show-rc --compression lz4 --exclude-caches "
|
|
|
|
# Setting this, so the repo does not need to be given on the commandline:
|
|
export BORG_REPO="ssh://{{ borg_username}}@${1}/opt/backups/{{ borg_username }}/backup"
|
|
|
|
# some helpers and error handling:
|
|
info() { printf "\n%s %s\n\n" "$( date )" "$*" >&2; }
|
|
trap 'echo $( date ) Backup interrupted >&2; exit 2' INT TERM
|
|
|
|
info "Starting backup"
|
|
|
|
# This avoids UI prompts when first accessing the remote repository
|
|
export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=1
|
|
|
|
# Backup the most important directories into an archive named after
|
|
# the machine this script is currently running on:
|
|
${BORG_CREATE} \
|
|
{% for item in borg_backup_excludes + borg_backup_excludes_extra -%}
|
|
--exclude '{{ item }}' \
|
|
{% endfor -%}
|
|
::'{hostname}-filesystem-{now}' \
|
|
{% for item in borg_backup_dirs + borg_backup_dirs_extra -%}
|
|
{{ item }} {{ '\\' if not loop.last }}
|
|
{% endfor -%}
|
|
|
|
backup_exit=$?
|
|
|
|
for f in $(shopt -s nullglob; echo /etc/borg-streams/*)
|
|
do
|
|
stream_name=$(basename $f)
|
|
info "Backing up stream archive $stream_name"
|
|
bash $f | ${BORG_CREATE} --stdin-name ${stream_name} \
|
|
::"{hostname}-${stream_name}-{now}" -
|
|
|
|
_status=( "${PIPESTATUS[@]}" )
|
|
if [[ ${_status[0]} -ne 0 ]]; then
|
|
info "Streaming script ${f} failed!"
|
|
info "Note that problems in the ssh connectivity might cause the streaming script to fail. You may need to check both halves of the streaming backup."
|
|
stream_exit=${_status[0]}
|
|
elif [[ ${_status[1]} -ne 0 ]]; then
|
|
info "Borg failed (rc: ${_status[1]})!"
|
|
stream_exit=${_status[1]}
|
|
else
|
|
stream_exit=0
|
|
fi
|
|
(( backup_exit = backup_exit || stream_exit ))
|
|
done
|
|
|
|
if [ ${backup_exit} -eq 0 ]; then
|
|
info "Backup finished successfully"
|
|
else
|
|
info "Backup finished with errors"
|
|
if [ ${BORG_UNDER_CRON:-0} -eq 1 ]; then
|
|
echo "Backups failed on host $(hostname) at $(date)." | \
|
|
mail -s "ACTION REQUIRED: Backup failed on $(hostname)" infra-root@openstack.org
|
|
fi
|
|
fi
|
|
|
|
|
|
exit ${backup_exit}
|
|
|