From fa8916f5bcc8cbf064a387569e2630b7bbf0b49b Mon Sep 17 00:00:00 2001
From: "Markin, Sergiy" <sergiy.markin@att.com>
Date: Mon, 23 Jan 2023 11:20:49 -0600
Subject: [PATCH] [helm-toolkit] Added a random delay to remote backup
 operations

This PS adds a random delay up to 300 seconds to remote backup upload
and download actions to spread the network load in time. Backup process
failure may happen if many sites are pushing their backups at the same
time. It was OK previously but now with added remote bakup sha256
checksum verification we need to download the backup we just uploaded.
So the network load already doubled. And this PS mitigates the impact
of that.

Change-Id: Ibc2a8f8287e20aeb56ad1f9c604b47db2d0eb06c
---
 helm-toolkit/Chart.yaml                                |  2 +-
 .../scripts/db-backup-restore/_backup_main.sh.tpl      | 10 ++++++++++
 releasenotes/notes/helm-toolkit.yaml                   |  1 +
 3 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/helm-toolkit/Chart.yaml b/helm-toolkit/Chart.yaml
index 611b87d83..c8897c355 100644
--- a/helm-toolkit/Chart.yaml
+++ b/helm-toolkit/Chart.yaml
@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v1.0.0
 description: OpenStack-Helm Helm-Toolkit
 name: helm-toolkit
-version: 0.2.50
+version: 0.2.51
 home: https://docs.openstack.org/openstack-helm
 icon: https://www.openstack.org/themes/openstack/images/project-mascots/OpenStack-Helm/OpenStack_Project_OpenStackHelm_vertical.png
 sources:
diff --git a/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl b/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
index 687851eb4..9597d3421 100755
--- a/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
+++ b/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
@@ -213,6 +213,11 @@ send_to_remote_server() {
     fi
   fi
 
+  # load balance delay
+  DELAY=$((1 + ${RANDOM} % 300))
+  echo "Sleeping for ${DELAY} seconds to spread the load in time..."
+  sleep ${DELAY}
+
   # Create an object to store the file
   openstack object create --name $FILE $CONTAINER_NAME $FILEPATH/$FILE
   if [[ $? -ne 0 ]]; then
@@ -226,6 +231,11 @@ send_to_remote_server() {
     return 2
   fi
 
+  # load balance delay
+  DELAY=$((1 + ${RANDOM} % 300))
+  echo "Sleeping for ${DELAY} seconds to spread the load in time..."
+  sleep ${DELAY}
+
   # Calculation remote file SHA256 hash
   REMOTE_FILE=$(mktemp -p /tmp)
   openstack object save --file ${REMOTE_FILE} $CONTAINER_NAME $FILE
diff --git a/releasenotes/notes/helm-toolkit.yaml b/releasenotes/notes/helm-toolkit.yaml
index 1aa5e41a5..918baf4e0 100644
--- a/releasenotes/notes/helm-toolkit.yaml
+++ b/releasenotes/notes/helm-toolkit.yaml
@@ -57,4 +57,5 @@ helm-toolkit:
   - 0.2.48 Added verify_databases_backup_archives function call to backup process and added remote backup sha256 hash verification
   - 0.2.49 Moved RabbitMQ Guest Admin removal to init
   - 0.2.50 Allow tls for external ingress without specifying key and crt
+  - 0.2.51 Added a random delay up to 300 seconds to remote backup upload/download for load spreading purpose
 ...