borg-backup-server: run a weekly backup verification

This checks the backup archives and alerts us if anything seems wrong.
This will take a few hours, so we run once a week.

Change-Id: I832c0d29a37df94d4bf2704c59bb3f8d855c3cc8
This commit is contained in:
Ian Wienand 2021-02-10 12:25:21 +11:00
parent 1a3ae8cdd8
commit 0d01d941b1
4 changed files with 54 additions and 0 deletions

View File

@ -0,0 +1,22 @@
#!/bin/bash
pushd /opt/backups
for u in borg-*; do
BORG_REPO=/opt/backups/$u/backup
sudo BORG_RELOCATED_REPO_ACCESS_IS_OK=y BORG_REPO=${BORG_REPO} -u ${u} -s <<'EOF'
echo "$(date) Verifying ${BORG_REPO} ..."
/opt/borg/bin/borg check --verify-data
if [[ $? -ne 0 ]]; then
echo "$(date) *** Verification failed"
echo "Inconsistency found in backup ${BORG_REPO} on $(hostname) at $(date)" |
mail -s "ACTION REQUIRED: Backup inconsistency: ${BORG_REPO}" infra-root@openstack.org
else
echo "$(date) ... done"
echo
fi
EOF
done

View File

@ -31,6 +31,29 @@
minute: '0'
hour: '0'
- name: Install backup verification
copy:
src: 'verify-borg-backups.sh'
dest: '/usr/local/bin/verify-borg-backups'
owner: root
group: root
mode: '0755'
- name: Run backup verification
cron:
name: verify-borg-backups
state: present
job: '/usr/local/bin/verify-borg-backups &> /var/log/verify-borg-backups.log'
minute: '0'
hour: '0'
weekday: '0'
- name: Rotate verification logs
include_role:
name: logrotate
vars:
logrotate_file_name: '/var/log/verify-borg-backups.log'
- name: Build all borg users from backup hosts
set_fact:
borg_users: '{{ borg_users }} + [ {{ hostvars[item]["borg_user"] }} ]'

View File

@ -96,3 +96,11 @@ def test_borg_server_prune(host):
cmd = host.run('echo "prune" | /usr/local/bin/prune-borg-backups &> /var/log/prune-borg-backups.log')
assert cmd.succeeded
def test_borg_server_verify(host):
hostname = host.backend.get_hostname()
if hostname.startswith('borg-backup-test'):
pytest.skip()
cmd = host.run('/usr/local/bin/verify-borg-backups &> /var/log/verify-borg-backups.log')
assert cmd.succeeded

View File

@ -360,6 +360,7 @@
borg-backup01.region.provider.opendev.org:
host_copy_output:
'/var/log/prune-borg-backups.log': logs
'/var/log/verify-borg-backups.log': logs
borg-backup-test01.opendev.org:
host_copy_output:
'/var/log/borg-backup-borg-backup01.region.provider.opendev.org.log': logs