borg-backup-server: run a weekly backup verification
This checks the backup archives and alerts us if anything seems wrong. This will take a few hours, so we run once a week. Change-Id: I832c0d29a37df94d4bf2704c59bb3f8d855c3cc8
This commit is contained in:
parent
1a3ae8cdd8
commit
0d01d941b1
@ -0,0 +1,22 @@
|
||||
#!/bin/bash
|
||||
|
||||
pushd /opt/backups
|
||||
|
||||
for u in borg-*; do
|
||||
BORG_REPO=/opt/backups/$u/backup
|
||||
|
||||
sudo BORG_RELOCATED_REPO_ACCESS_IS_OK=y BORG_REPO=${BORG_REPO} -u ${u} -s <<'EOF'
|
||||
|
||||
echo "$(date) Verifying ${BORG_REPO} ..."
|
||||
/opt/borg/bin/borg check --verify-data
|
||||
if [[ $? -ne 0 ]]; then
|
||||
echo "$(date) *** Verification failed"
|
||||
echo "Inconsistency found in backup ${BORG_REPO} on $(hostname) at $(date)" |
|
||||
mail -s "ACTION REQUIRED: Backup inconsistency: ${BORG_REPO}" infra-root@openstack.org
|
||||
else
|
||||
echo "$(date) ... done"
|
||||
echo
|
||||
fi
|
||||
|
||||
EOF
|
||||
done
|
@ -31,6 +31,29 @@
|
||||
minute: '0'
|
||||
hour: '0'
|
||||
|
||||
- name: Install backup verification
|
||||
copy:
|
||||
src: 'verify-borg-backups.sh'
|
||||
dest: '/usr/local/bin/verify-borg-backups'
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
|
||||
- name: Run backup verification
|
||||
cron:
|
||||
name: verify-borg-backups
|
||||
state: present
|
||||
job: '/usr/local/bin/verify-borg-backups &> /var/log/verify-borg-backups.log'
|
||||
minute: '0'
|
||||
hour: '0'
|
||||
weekday: '0'
|
||||
|
||||
- name: Rotate verification logs
|
||||
include_role:
|
||||
name: logrotate
|
||||
vars:
|
||||
logrotate_file_name: '/var/log/verify-borg-backups.log'
|
||||
|
||||
- name: Build all borg users from backup hosts
|
||||
set_fact:
|
||||
borg_users: '{{ borg_users }} + [ {{ hostvars[item]["borg_user"] }} ]'
|
||||
|
@ -96,3 +96,11 @@ def test_borg_server_prune(host):
|
||||
|
||||
cmd = host.run('echo "prune" | /usr/local/bin/prune-borg-backups &> /var/log/prune-borg-backups.log')
|
||||
assert cmd.succeeded
|
||||
|
||||
def test_borg_server_verify(host):
|
||||
hostname = host.backend.get_hostname()
|
||||
if hostname.startswith('borg-backup-test'):
|
||||
pytest.skip()
|
||||
|
||||
cmd = host.run('/usr/local/bin/verify-borg-backups &> /var/log/verify-borg-backups.log')
|
||||
assert cmd.succeeded
|
||||
|
@ -360,6 +360,7 @@
|
||||
borg-backup01.region.provider.opendev.org:
|
||||
host_copy_output:
|
||||
'/var/log/prune-borg-backups.log': logs
|
||||
'/var/log/verify-borg-backups.log': logs
|
||||
borg-backup-test01.opendev.org:
|
||||
host_copy_output:
|
||||
'/var/log/borg-backup-borg-backup01.region.provider.opendev.org.log': logs
|
||||
|
Loading…
x
Reference in New Issue
Block a user