openstack/openstack-helm-infra
Code Issues Proposed changes

[backups] Database backups

Browse Source 
This PS resolves several issues in database backup script in HTK chart:

- decreases random delay before uploading remote backup to up to 30s
- removes additional random delay before remote backup verification
- switches remote backup verification protocol from sha256 to md5

The main goal for the changes above is decreasing network load on remote
backup storages by eliminating the need of remote file download right
after uploading in order to be able to calculate sha256 checksum.

Change-Id: Ic01a37d8814283a2e9a11dac94d6909d34edc937
This commit is contained in:
Sergiy Markin 2023-03-01 18:16:41 +00:00
parent 150dcdcf9a
commit a7cd689280
3 changed files with 12 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
helm-toolkit/Chart.yaml
View File

@ -15,7 +15,7 @@ apiVersion: v1
appVersion: v1.0.0
description: OpenStack-Helm Helm-Toolkit
name: helm-toolkit
version: 0.2.51
version: 0.2.52
home: https://docs.openstack.org/openstack-helm
icon: https://www.openstack.org/themes/openstack/images/project-mascots/OpenStack-Helm/OpenStack_Project_OpenStackHelm_vertical.png
sources:

34
helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
View File

@ -214,7 +214,7 @@ send_to_remote_server() {
fi
# load balance delay
DELAY=$((1 + ${RANDOM} % 300))
DELAY=$((1 + ${RANDOM} % 30))
echo "Sleeping for ${DELAY} seconds to spread the load in time..."
sleep ${DELAY}
@ -231,31 +231,17 @@ send_to_remote_server() {
return 2
fi
# load balance delay
DELAY=$((1 + ${RANDOM} % 300))
echo "Sleeping for ${DELAY} seconds to spread the load in time..."
sleep ${DELAY}
# Calculation remote file SHA256 hash
REMOTE_FILE=$(mktemp -p /tmp)
openstack object save --file ${REMOTE_FILE} $CONTAINER_NAME $FILE
if [[ $? -ne 0 ]]; then
log WARN "${DB_NAME}_backup" "Unable to save container object $FILE for SHA256 hash verification."
rm -rf ${REMOTE_FILE}
return 1
fi
# Remote backup verification
SHA256_REMOTE=$(cat ${REMOTE_FILE} | sha256sum | awk '{print $1}')
SHA256_LOCAL=$(cat ${FILEPATH}/${FILE} | sha256sum | awk '{print $1}')
log INFO "${DB_NAME}_backup" "Calculated SHA256 hashes for the file $FILE in container $CONTAINER_NAME."
log INFO "${DB_NAME}_backup" "Local SHA256 hash is ${SHA256_LOCAL}."
log INFO "${DB_NAME}_backup" "Remote SHA256 hash is ${SHA256_REMOTE}."
if [[ "${SHA256_LOCAL}" == "${SHA256_REMOTE}" ]]; then
log INFO "${DB_NAME}_backup" "The local backup & remote backup SHA256 hash values are matching for file $FILE in container $CONTAINER_NAME."
MD5_REMOTE=$(openstack object show $CONTAINER_NAME $FILE -f json | jq -r ".etag")
MD5_LOCAL=$(cat ${FILEPATH}/${FILE} | md5sum | awk '{print $1}')
log INFO "${DB_NAME}_backup" "Obtained MD5 hash for the file $FILE in container $CONTAINER_NAME."
log INFO "${DB_NAME}_backup" "Local MD5 hash is ${MD5_LOCAL}."
log INFO "${DB_NAME}_backup" "Remote MD5 hash is ${MD5_REMOTE}."
if [[ "${MD5_LOCAL}" == "${MD5_REMOTE}" ]]; then
log INFO "${DB_NAME}_backup" "The local backup & remote backup MD5 hash values are matching for file $FILE in container $CONTAINER_NAME."
else
log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup sha256 hash values"
return 1
log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup MD5 hash values"
return 2
fi
rm -rf ${REMOTE_FILE}

1
releasenotes/notes/helm-toolkit.yaml
View File

@ -58,4 +58,5 @@ helm-toolkit:
- 0.2.49 Moved RabbitMQ Guest Admin removal to init
- 0.2.50 Allow tls for external ingress without specifying key and crt
- 0.2.51 Added a random delay up to 300 seconds to remote backup upload/download for load spreading purpose
- 0.2.52 Decreased random delay to up to 30 seconds and switched remote backup verification protocol to md5
...