[backups] Add throttlling of remote backups

This PS adds a possibility to limit (to throttle) the number of
simultaneously uploaded backups while keeping the logic on the client
side using flag files on remote side.

Also this PS uplifts HTK to 0.2.59 and bumps up helm binary version to
v3.13.2.

Change-Id: I5f07334d1f48761929bde3e3b347271f2c67ac63

charts/etcdctl-utility/Chart.yaml

@@ -13,4 +13,4 @@
 apiVersion: v1
 description: etcdctl Client
 name: etcdctl-utility
-version: 0.1.4
+version: 0.1.5

charts/etcdctl-utility/templates/bin/utility/_etcd_ondemand_job.sh.tpl

@@ -117,6 +117,31 @@ spec:
                 secretKeyRef:
                   key: REMOTE_BACKUP_SEND_DELAY_MAX
                   name: ${ETCD_CONF_SECRET}
+            - name: THROTTLE_BACKUPS_ENABLED
+              valueFrom:
+                secretKeyRef:
+                  key: THROTTLE_BACKUPS_ENABLED
+                  name: ${ETCD_CONF_SECRET}
+            - name: THROTTLE_LIMIT
+              valueFrom:
+                secretKeyRef:
+                  key: THROTTLE_LIMIT
+                  name: ${ETCD_CONF_SECRET}
+            - name: THROTTLE_LOCK_EXPIRE_AFTER
+              valueFrom:
+                secretKeyRef:
+                  key: THROTTLE_LOCK_EXPIRE_AFTER
+                  name: ${ETCD_CONF_SECRET}
+            - name: THROTTLE_RETRY_AFTER
+              valueFrom:
+                secretKeyRef:
+                  key: THROTTLE_RETRY_AFTER
+                  name: ${ETCD_CONF_SECRET}
+            - name: THROTTLE_CONTAINER_NAME
+              valueFrom:
+                secretKeyRef:
+                  key: THROTTLE_CONTAINER_NAME
+                  name: ${ETCD_CONF_SECRET}
 EOF
 
 if $ETCD_REMOTE_BACKUP_ENABLED; then

charts/mysqlclient-utility/Chart.yaml

@@ -13,4 +13,4 @@
 apiVersion: v1
 description: Porthole MySql Client
 name: mysqlclient-utility
-version: 0.1.7
+version: 0.1.8

charts/mysqlclient-utility/templates/bin/utility/_mariadb_ondemand_job.sh.tpl

@@ -146,6 +146,31 @@ spec:
                 secretKeyRef:
                   key: REMOTE_BACKUP_SEND_DELAY_MAX
                   name: ${MARIADB_CONF_SECRET}
+            - name: THROTTLE_BACKUPS_ENABLED
+              valueFrom:
+                secretKeyRef:
+                  key: THROTTLE_BACKUPS_ENABLED
+                  name: ${MARIADB_CONF_SECRET}
+            - name: THROTTLE_LIMIT
+              valueFrom:
+                secretKeyRef:
+                  key: THROTTLE_LIMIT
+                  name: ${MARIADB_CONF_SECRET}
+            - name: THROTTLE_LOCK_EXPIRE_AFTER
+              valueFrom:
+                secretKeyRef:
+                  key: THROTTLE_LOCK_EXPIRE_AFTER
+                  name: ${MARIADB_CONF_SECRET}
+            - name: THROTTLE_RETRY_AFTER
+              valueFrom:
+                secretKeyRef:
+                  key: THROTTLE_RETRY_AFTER
+                  name: ${MARIADB_CONF_SECRET}
+            - name: THROTTLE_CONTAINER_NAME
+              valueFrom:
+                secretKeyRef:
+                  key: THROTTLE_CONTAINER_NAME
+                  name: ${MARIADB_CONF_SECRET}
 EOF
 
 if $MARIADB_REMOTE_BACKUP_ENABLED; then

charts/postgresql-utility/Chart.yaml

@@ -14,4 +14,4 @@
 apiVersion: v1
 description: PostgreSQL Client
 name: postgresql-utility
-version: 0.1.3
+version: 0.1.4

charts/postgresql-utility/templates/bin/utility/_pg_ondemand_job.sh.tpl

@@ -130,6 +130,31 @@ spec:
                 secretKeyRef:
                   key: REMOTE_BACKUP_SEND_DELAY_MAX
                   name: ${POSTGRESQL_CONF_SECRET}
+            - name: THROTTLE_BACKUPS_ENABLED
+              valueFrom:
+                secretKeyRef:
+                  key: THROTTLE_BACKUPS_ENABLED
+                  name: ${POSTGRESQL_CONF_SECRET}
+            - name: THROTTLE_LIMIT
+              valueFrom:
+                secretKeyRef:
+                  key: THROTTLE_LIMIT
+                  name: ${POSTGRESQL_CONF_SECRET}
+            - name: THROTTLE_LOCK_EXPIRE_AFTER
+              valueFrom:
+                secretKeyRef:
+                  key: THROTTLE_LOCK_EXPIRE_AFTER
+                  name: ${POSTGRESQL_CONF_SECRET}
+            - name: THROTTLE_RETRY_AFTER
+              valueFrom:
+                secretKeyRef:
+                  key: THROTTLE_RETRY_AFTER
+                  name: ${POSTGRESQL_CONF_SECRET}
+            - name: THROTTLE_CONTAINER_NAME
+              valueFrom:
+                secretKeyRef:
+                  key: THROTTLE_CONTAINER_NAME
+                  name: ${POSTGRESQL_CONF_SECRET}
 EOF
 
 if $POSTGRESQL_REMOTE_BACKUP_ENABLED; then

tools/helm_install.sh

@@ -17,7 +17,7 @@
 set -x
 
 HELM=$1
-HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.12.2-linux-amd64.tar.gz"}
+HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz"}
 
 
 function install_helm_binary {

tools/helm_tk.sh

@@ -17,7 +17,7 @@
 set -eux
 
 HTK_REPO=${HTK_REPO:-"https://opendev.org/openstack/openstack-helm-infra.git"}
-HTK_STABLE_COMMIT=${HTK_COMMIT:-"97ce6d7d8e9a090c748800d69a57bbd9af698b60"}
+HTK_STABLE_COMMIT=${HTK_COMMIT:-"13c1d8cd3866bb026cce6e5d80555edfd0afa845"}
 
 TMP_DIR=$(mktemp -d)