[backups] Add throttlling of remote backups
This PS adds a possibility to limit (to throttle) the number of simultaneously uploaded backups while keeping the logic on the client side using flag files on remote side. The main idea is to have an ability to limit number of simultaneous remote backups upload sessions. Change-Id: I5464004d4febfbe20df9cd41ca62ceb9fd6f0c0d
This commit is contained in:
Sergiy Markin
parent
f66c924b2f
commit
13c1d8cd38
@ -15,7 +15,7 @@ apiVersion: v1
|
||||
appVersion: v1.0.0
|
||||
description: OpenStack-Helm Helm-Toolkit
|
||||
name: helm-toolkit
|
||||
version: 0.2.58
|
||||
version: 0.2.59
|
||||
home: https://docs.openstack.org/openstack-helm
|
||||
icon: https://www.openstack.org/themes/openstack/images/project-mascots/OpenStack-Helm/OpenStack_Project_OpenStackHelm_vertical.png
|
||||
sources:
|
||||
|
||||
@ -49,6 +49,13 @@
|
||||
# A random number between min and max delay is generated
|
||||
# to set the delay.
|
||||
#
|
||||
# RGW backup throttle limits variables:
|
||||
# export THROTTLE_BACKUPS_ENABLED Boolean variableto control backup functionality
|
||||
# export THROTTLE_LIMIT Number of simultaneous RGW upload sessions
|
||||
# export THROTTLE_LOCK_EXPIRE_AFTER Time in seconds to expire flag file is orphaned
|
||||
# export THROTTLE_RETRY_AFTER Time in seconds to wait before retry
|
||||
# export THROTTLE_CONTAINER_NAME Name of RGW container to place flag falies into
|
||||
#
|
||||
# The database-specific functions that need to be implemented are:
|
||||
# dump_databases_to_directory <directory> <err_logfile> [scope]
|
||||
# where:
|
||||
@ -84,8 +91,10 @@
|
||||
# specified by the "LOCAL_DAYS_TO_KEEP" variable.
|
||||
# 4) Removing remote backup tarballs (from the remote gateway) which are older
|
||||
# than the number of days specified by the "REMOTE_DAYS_TO_KEEP" variable.
|
||||
# 5) Controlling remote storage gateway load from client side and throttling it
|
||||
# by using a dedicated RGW container to store flag files defining upload session
|
||||
# in progress
|
||||
#
|
||||
|
||||
# Note: not using set -e in this script because more elaborate error handling
|
||||
# is needed.
|
||||
|
||||
@ -218,6 +227,113 @@ send_to_remote_server() {
|
||||
echo "Sleeping for ${DELAY} seconds to spread the load in time..."
|
||||
sleep ${DELAY}
|
||||
|
||||
#---------------------------------------------------------------------------
|
||||
# Remote backup throttling
|
||||
export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
|
||||
if $THROTTLE_BACKUPS_ENABLED; then
|
||||
# Remove Quotes from the constants which were added due to reading
|
||||
# from secret.
|
||||
export THROTTLE_LIMIT=$(echo $THROTTLE_LIMIT | sed 's/"//g')
|
||||
export THROTTLE_LOCK_EXPIRE_AFTER=$(echo $THROTTLE_LOCK_EXPIRE_AFTER | sed 's/"//g')
|
||||
export THROTTLE_RETRY_AFTER=$(echo $THROTTLE_RETRY_AFTER | sed 's/"//g')
|
||||
export THROTTLE_CONTAINER_NAME=$(echo $THROTTLE_CONTAINER_NAME | sed 's/"//g')
|
||||
|
||||
# load balance delay
|
||||
RESULT=$(openstack container list 2>&1)
|
||||
|
||||
if [[ $? -eq 0 ]]; then
|
||||
echo $RESULT | grep $THROTTLE_CONTAINER_NAME
|
||||
if [[ $? -ne 0 ]]; then
|
||||
# Find the swift URL from the keystone endpoint list
|
||||
SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}')
|
||||
if [[ $? -ne 0 ]]; then
|
||||
log WARN "${DB_NAME}_backup" "Unable to get object-store enpoints from keystone catalog."
|
||||
return 2
|
||||
fi
|
||||
|
||||
# Get a token from keystone
|
||||
TOKEN=$(openstack token issue -f value -c id)
|
||||
if [[ $? -ne 0 ]]; then
|
||||
log WARN "${DB_NAME}_backup" "Unable to get keystone token."
|
||||
return 2
|
||||
fi
|
||||
|
||||
# Create the container
|
||||
RES_FILE=$(mktemp -p /tmp)
|
||||
curl -g -i -X PUT ${SWIFT_URL}/${THROTTLE_CONTAINER_NAME} \
|
||||
-H "X-Auth-Token: ${TOKEN}" \
|
||||
-H "X-Storage-Policy: ${STORAGE_POLICY}" 2>&1 > $RES_FILE
|
||||
|
||||
if [[ $? -ne 0 || $(grep "HTTP" $RES_FILE | awk '{print $2}') -ge 400 ]]; then
|
||||
log WARN "${DB_NAME}_backup" "Unable to create container ${THROTTLE_CONTAINER_NAME}"
|
||||
cat $RES_FILE
|
||||
rm -f $RES_FILE
|
||||
return 2
|
||||
fi
|
||||
rm -f $RES_FILE
|
||||
|
||||
swift stat $THROTTLE_CONTAINER_NAME
|
||||
if [[ $? -ne 0 ]]; then
|
||||
log WARN "${DB_NAME}_backup" "Unable to retrieve container ${THROTTLE_CONTAINER_NAME} details after creation."
|
||||
return 2
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo $RESULT | grep -E "HTTP 401|HTTP 403"
|
||||
if [[ $? -eq 0 ]]; then
|
||||
log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}"
|
||||
return 1
|
||||
else
|
||||
echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable|HTTP 50"
|
||||
if [[ $? -eq 0 ]]; then
|
||||
log WARN "${DB_NAME}_backup" "Could not reach the RGW: ${RESULT}"
|
||||
# In this case, keystone or the site/node may be temporarily down.
|
||||
# Return slightly different error code so the calling code can retry
|
||||
return 2
|
||||
else
|
||||
log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
|
||||
log INFO "${DB_NAME}_backup" "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
|
||||
while [[ ${NUMBER_OF_SESSIONS} -ge ${THROTTLE_LIMIT} ]]
|
||||
do
|
||||
log INFO "${DB_NAME}_backup" "Current number of active uploads is ${NUMBER_OF_SESSIONS}>=${THROTTLE_LIMIT}!"
|
||||
log INFO "${DB_NAME}_backup" "Retrying in ${THROTTLE_RETRY_AFTER} seconds...."
|
||||
sleep ${THROTTLE_RETRY_AFTER}
|
||||
NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
|
||||
log INFO "${DB_NAME}_backup" "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
|
||||
done
|
||||
|
||||
# Create a lock file in THROTTLE_CONTAINER
|
||||
THROTTLE_FILEPATH=$(mktemp -d)
|
||||
THROTTLE_FILE=${CONTAINER_NAME}.lock
|
||||
date +%s > $THROTTLE_FILEPATH/$THROTTLE_FILE
|
||||
|
||||
# Create an object to store the file
|
||||
openstack object create --name $THROTTLE_FILE $THROTTLE_CONTAINER_NAME $THROTTLE_FILEPATH/$THROTTLE_FILE
|
||||
if [[ $? -ne 0 ]]; then
|
||||
log WARN "${DB_NAME}_backup" "Cannot create throttle container object ${THROTTLE_FILE}!"
|
||||
return 2
|
||||
fi
|
||||
|
||||
swift post $THROTTLE_CONTAINER_NAME $THROTTLE_FILE -H "X-Delete-After:${THROTTLE_LOCK_EXPIRE_AFTER}"
|
||||
if [[ $? -ne 0 ]]; then
|
||||
log WARN "${DB_NAME}_backup" "Cannot set throttle container object ${THROTTLE_FILE} expiration header!"
|
||||
return 2
|
||||
fi
|
||||
openstack object show $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
|
||||
if [[ $? -ne 0 ]]; then
|
||||
log WARN "${DB_NAME}_backup" "Unable to retrieve throttle container object $THROTTLE_FILE after creation."
|
||||
return 2
|
||||
fi
|
||||
fi
|
||||
|
||||
#---------------------------------------------------------------------------
|
||||
|
||||
# Create an object to store the file
|
||||
openstack object create --name $FILE $CONTAINER_NAME $FILEPATH/$FILE
|
||||
if [[ $? -ne 0 ]]; then
|
||||
@ -245,6 +361,24 @@ send_to_remote_server() {
|
||||
fi
|
||||
rm -f ${REMOTE_FILE}
|
||||
|
||||
#---------------------------------------------------------------------------
|
||||
# Remote backup throttling
|
||||
export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
|
||||
if $THROTTLE_BACKUPS_ENABLED; then
|
||||
# Remove flag file
|
||||
# Delete an object to remove the flag file
|
||||
openstack object delete $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
|
||||
if [[ $? -ne 0 ]]; then
|
||||
log WARN "${DB_NAME}_backup" "Cannot delete throttle container object ${THROTTLE_FILE}"
|
||||
return 0
|
||||
else
|
||||
log INFO "${DB_NAME}_backup" "The throttle container object ${THROTTLE_FILE} has been successfully removed."
|
||||
fi
|
||||
rm -f ${THROTTLE_FILEPATH}/${THROTTLE_FILE}
|
||||
fi
|
||||
|
||||
#---------------------------------------------------------------------------
|
||||
|
||||
log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully."
|
||||
return 0
|
||||
}
|
||||
|
||||
@ -15,7 +15,7 @@ apiVersion: v1
|
||||
appVersion: v10.6.14
|
||||
description: OpenStack-Helm MariaDB backups
|
||||
name: mariadb-backup
|
||||
version: 0.0.3
|
||||
version: 0.0.4
|
||||
home: https://mariadb.com/kb/en/
|
||||
icon: http://badges.mariadb.org/mariadb-badge-180x60.png
|
||||
sources:
|
||||
|
||||
@ -132,6 +132,16 @@ spec:
|
||||
value: {{ .Values.conf.backup.remote_backup.delay_range.min | quote }}
|
||||
- name: MAX_DELAY_SEND_BACKUP_TO_REMOTE
|
||||
value: {{ .Values.conf.backup.remote_backup.delay_range.max | quote }}
|
||||
- name: THROTTLE_BACKUPS_ENABLED
|
||||
value: "{{ .Values.conf.backup.remote_backup.throttle_backups.enabled }}"
|
||||
- name: THROTTLE_LIMIT
|
||||
value: {{ .Values.conf.backup.remote_backup.throttle_backups.sessions_limit | quote }}
|
||||
- name: THROTTLE_LOCK_EXPIRE_AFTER
|
||||
value: {{ .Values.conf.backup.remote_backup.throttle_backups.lock_expire_after | quote }}
|
||||
- name: THROTTLE_RETRY_AFTER
|
||||
value: {{ .Values.conf.backup.remote_backup.throttle_backups.retry_after | quote }}
|
||||
- name: THROTTLE_CONTAINER_NAME
|
||||
value: {{ .Values.conf.backup.remote_backup.throttle_backups.container_name | quote }}
|
||||
{{- with $env := dict "ksUserSecret" $envAll.Values.secrets.identity.mariadb }}
|
||||
{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 16 }}
|
||||
{{- end }}
|
||||
|
||||
@ -26,5 +26,10 @@ data:
|
||||
REMOTE_BACKUP_RETRIES: {{ $envAll.Values.conf.backup.remote_backup.number_of_retries | quote | b64enc }}
|
||||
REMOTE_BACKUP_SEND_DELAY_MIN: {{ $envAll.Values.conf.backup.remote_backup.delay_range.min | quote | b64enc }}
|
||||
REMOTE_BACKUP_SEND_DELAY_MAX: {{ $envAll.Values.conf.backup.remote_backup.delay_range.max | quote | b64enc }}
|
||||
THROTTLE_BACKUPS_ENABLED: {{ $envAll.Values.conf.backup.remote_backup.throttle_backups.enabled | quote | b64enc }}
|
||||
THROTTLE_LIMIT: {{ $envAll.Values.conf.backup.remote_backup.throttle_backups.sessions_limit | quote | b64enc }}
|
||||
THROTTLE_LOCK_EXPIRE_AFTER: {{ $envAll.Values.conf.backup.remote_backup.throttle_backups.lock_expire_after | quote | b64enc }}
|
||||
THROTTLE_RETRY_AFTER: {{ $envAll.Values.conf.backup.remote_backup.throttle_backups.retry_after | quote | b64enc }}
|
||||
THROTTLE_CONTAINER_NAME: {{ $envAll.Values.conf.backup.remote_backup.throttle_backups.container_name | quote | b64enc }}
|
||||
...
|
||||
{{- end }}
|
||||
|
||||
@ -235,6 +235,12 @@ conf:
|
||||
delay_range:
|
||||
min: 30
|
||||
max: 60
|
||||
throttle_backups:
|
||||
enabled: false
|
||||
sessions_limit: 480
|
||||
lock_expire_after: 7200
|
||||
retry_after: 3600
|
||||
container_name: throttle-backups-manager
|
||||
|
||||
secrets:
|
||||
identity:
|
||||
|
||||
@ -15,7 +15,7 @@ apiVersion: v1
|
||||
appVersion: v10.6.7
|
||||
description: OpenStack-Helm MariaDB
|
||||
name: mariadb
|
||||
version: 0.2.37
|
||||
version: 0.2.38
|
||||
home: https://mariadb.com/kb/en/
|
||||
icon: http://badges.mariadb.org/mariadb-badge-180x60.png
|
||||
sources:
|
||||
|
||||
@ -132,6 +132,16 @@ spec:
|
||||
value: {{ .Values.conf.backup.remote_backup.delay_range.min | quote }}
|
||||
- name: MAX_DELAY_SEND_BACKUP_TO_REMOTE
|
||||
value: {{ .Values.conf.backup.remote_backup.delay_range.max | quote }}
|
||||
- name: THROTTLE_BACKUPS_ENABLED
|
||||
value: "{{ .Values.conf.backup.remote_backup.throttle_backups.enabled }}"
|
||||
- name: THROTTLE_LIMIT
|
||||
value: {{ .Values.conf.backup.remote_backup.throttle_backups.sessions_limit | quote }}
|
||||
- name: THROTTLE_LOCK_EXPIRE_AFTER
|
||||
value: {{ .Values.conf.backup.remote_backup.throttle_backups.lock_expire_after | quote }}
|
||||
- name: THROTTLE_RETRY_AFTER
|
||||
value: {{ .Values.conf.backup.remote_backup.throttle_backups.retry_after | quote }}
|
||||
- name: THROTTLE_CONTAINER_NAME
|
||||
value: {{ .Values.conf.backup.remote_backup.throttle_backups.container_name | quote }}
|
||||
{{- with $env := dict "ksUserSecret" $envAll.Values.secrets.identity.mariadb }}
|
||||
{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 16 }}
|
||||
{{- end }}
|
||||
|
||||
@ -26,5 +26,10 @@ data:
|
||||
REMOTE_BACKUP_RETRIES: {{ $envAll.Values.conf.backup.remote_backup.number_of_retries | quote | b64enc }}
|
||||
REMOTE_BACKUP_SEND_DELAY_MIN: {{ $envAll.Values.conf.backup.remote_backup.delay_range.min | quote | b64enc }}
|
||||
REMOTE_BACKUP_SEND_DELAY_MAX: {{ $envAll.Values.conf.backup.remote_backup.delay_range.max | quote | b64enc }}
|
||||
THROTTLE_BACKUPS_ENABLED: {{ $envAll.Values.conf.backup.remote_backup.throttle_backups.enabled | quote | b64enc }}
|
||||
THROTTLE_LIMIT: {{ $envAll.Values.conf.backup.remote_backup.throttle_backups.sessions_limit | quote | b64enc }}
|
||||
THROTTLE_LOCK_EXPIRE_AFTER: {{ $envAll.Values.conf.backup.remote_backup.throttle_backups.lock_expire_after | quote | b64enc }}
|
||||
THROTTLE_RETRY_AFTER: {{ $envAll.Values.conf.backup.remote_backup.throttle_backups.retry_after | quote | b64enc }}
|
||||
THROTTLE_CONTAINER_NAME: {{ $envAll.Values.conf.backup.remote_backup.throttle_backups.container_name | quote | b64enc }}
|
||||
...
|
||||
{{- end }}
|
||||
|
||||
@ -356,6 +356,12 @@ conf:
|
||||
delay_range:
|
||||
min: 30
|
||||
max: 60
|
||||
throttle_backups:
|
||||
enabled: false
|
||||
sessions_limit: 480
|
||||
lock_expire_after: 7200
|
||||
retry_after: 3600
|
||||
container_name: throttle-backups-manager
|
||||
database:
|
||||
mysql_histfile: "/dev/null"
|
||||
my: |
|
||||
|
||||
@ -15,7 +15,7 @@ apiVersion: v1
|
||||
appVersion: v14.5
|
||||
description: OpenStack-Helm PostgreSQL
|
||||
name: postgresql
|
||||
version: 0.1.19
|
||||
version: 0.1.20
|
||||
home: https://www.postgresql.org
|
||||
sources:
|
||||
- https://github.com/postgres/postgres
|
||||
|
||||
@ -122,6 +122,16 @@ spec:
|
||||
value: {{ .Values.conf.backup.remote_backup.delay_range.min | quote }}
|
||||
- name: MAX_DELAY_SEND_BACKUP_TO_REMOTE
|
||||
value: {{ .Values.conf.backup.remote_backup.delay_range.max | quote }}
|
||||
- name: THROTTLE_BACKUPS_ENABLED
|
||||
value: "{{ .Values.conf.backup.remote_backup.throttle_backups.enabled }}"
|
||||
- name: THROTTLE_LIMIT
|
||||
value: {{ .Values.conf.backup.remote_backup.throttle_backups.sessions_limit | quote }}
|
||||
- name: THROTTLE_LOCK_EXPIRE_AFTER
|
||||
value: {{ .Values.conf.backup.remote_backup.throttle_backups.lock_expire_after | quote }}
|
||||
- name: THROTTLE_RETRY_AFTER
|
||||
value: {{ .Values.conf.backup.remote_backup.throttle_backups.retry_after | quote }}
|
||||
- name: THROTTLE_CONTAINER_NAME
|
||||
value: {{ .Values.conf.backup.remote_backup.throttle_backups.container_name | quote }}
|
||||
{{- with $env := dict "ksUserSecret" $envAll.Values.secrets.identity.postgresql }}
|
||||
{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 16 }}
|
||||
{{- end }}
|
||||
|
||||
@ -26,5 +26,10 @@ data:
|
||||
REMOTE_BACKUP_RETRIES: {{ $envAll.Values.conf.backup.remote_backup.number_of_retries | quote | b64enc }}
|
||||
REMOTE_BACKUP_SEND_DELAY_MIN: {{ $envAll.Values.conf.backup.remote_backup.delay_range.min | quote | b64enc }}
|
||||
REMOTE_BACKUP_SEND_DELAY_MAX: {{ $envAll.Values.conf.backup.remote_backup.delay_range.max | quote | b64enc }}
|
||||
THROTTLE_BACKUPS_ENABLED: {{ $envAll.Values.conf.backup.remote_backup.throttle_backups.enabled | quote | b64enc }}
|
||||
THROTTLE_LIMIT: {{ $envAll.Values.conf.backup.remote_backup.throttle_backups.sessions_limit | quote | b64enc }}
|
||||
THROTTLE_LOCK_EXPIRE_AFTER: {{ $envAll.Values.conf.backup.remote_backup.throttle_backups.lock_expire_after | quote | b64enc }}
|
||||
THROTTLE_RETRY_AFTER: {{ $envAll.Values.conf.backup.remote_backup.throttle_backups.retry_after | quote | b64enc }}
|
||||
THROTTLE_CONTAINER_NAME: {{ $envAll.Values.conf.backup.remote_backup.throttle_backups.container_name | quote | b64enc }}
|
||||
...
|
||||
{{- end }}
|
||||
|
||||
@ -328,6 +328,12 @@ conf:
|
||||
delay_range:
|
||||
min: 30
|
||||
max: 60
|
||||
throttle_backups:
|
||||
enabled: false
|
||||
sessions_limit: 480
|
||||
lock_expire_after: 7200
|
||||
retry_after: 3600
|
||||
container_name: throttle-backups-manager
|
||||
|
||||
exporter:
|
||||
queries:
|
||||
|
||||
@ -65,4 +65,5 @@ helm-toolkit:
|
||||
- 0.2.56 Expose S3 credentials from Rook bucket CRD secret
|
||||
- 0.2.57 Safer file removal
|
||||
- 0.2.58 Backups verification improvements
|
||||
- 0.2.59 Added throttling remote backups
|
||||
...
|
||||
|
||||
@ -3,4 +3,5 @@ mariadb-backup:
|
||||
- 0.0.1 Initial Chart
|
||||
- 0.0.2 Added staggered backups support
|
||||
- 0.0.3 Backups verification improvements
|
||||
- 0.0.4 Added throttling remote backups
|
||||
...
|
||||
|
||||
@ -53,4 +53,5 @@ mariadb:
|
||||
- 0.2.35 Update apparmor override
|
||||
- 0.2.36 Added staggered backups support
|
||||
- 0.2.37 Backups verification improvements
|
||||
- 0.2.38 Added throttling remote backups
|
||||
...
|
||||
|
||||
@ -20,4 +20,5 @@ postgresql:
|
||||
- 0.1.17 Added empty verify_databases_backup_archives() function implementation to match updated backup_databases() function in helm-toolkit
|
||||
- 0.1.18 Updated postgres to 14.5 and replaced deprecated config item wal_keep_segments with wal_keep_size
|
||||
- 0.1.19 Added staggered backups support
|
||||
- 0.1.20 Added throttling remote backups
|
||||
...
|
||||
|
||||
Loading…
Reference in New Issue
Block a user