fff85f029c
We have seen a case where the weekly verification run conflicted with an in-progress backup run. Make the verification step wait for up to an hour for the lock to allow backups to complete. Change-Id: Id87dd090c7cd652695ab0c4aa73477cf0d72c28d
27 lines
738 B
Bash
27 lines
738 B
Bash
#!/bin/bash
|
|
|
|
# When under cron, SHELL can bin /bin/sh which the "sudo -s" below
|
|
# obeys. Let's just use bash.
|
|
export SHELL=/bin/bash
|
|
|
|
pushd /opt/backups
|
|
|
|
for u in borg-*; do
|
|
BORG_REPO=/opt/backups/$u/backup
|
|
|
|
sudo BORG_RELOCATED_REPO_ACCESS_IS_OK=y BORG_REPO=${BORG_REPO} -u ${u} -s <<'EOF'
|
|
|
|
echo "$(date) Verifying ${BORG_REPO} ..."
|
|
/opt/borg/bin/borg check --lock-wait=3600 --verify-data
|
|
if [[ $? -ne 0 ]]; then
|
|
echo "$(date) *** Verification failed"
|
|
echo "Inconsistency found in backup ${BORG_REPO} on $(hostname) at $(date)" |
|
|
mail -s "ACTION REQUIRED: Backup inconsistency: ${BORG_REPO}" infra-root@openstack.org
|
|
else
|
|
echo "$(date) ... done"
|
|
echo
|
|
fi
|
|
|
|
EOF
|
|
done
|