From 025cc789f2dbd3fbbc010922293a6dc275a9dd55 Mon Sep 17 00:00:00 2001 From: Sergiy Markin Date: Tue, 12 Dec 2023 04:31:45 +0000 Subject: [PATCH] [backups] Add throttlling of remote backups This PS adds a possibility to limit (to throttle) the number of simultaneously uploaded backups while keeping the logic on the client side using flag files on remote side. Also this PS uplifts HTK to 0.2.59 and bumps up helm binary version to v3.13.2. Change-Id: I5f07334d1f48761929bde3e3b347271f2c67ac63 --- charts/etcdctl-utility/Chart.yaml | 2 +- .../bin/utility/_etcd_ondemand_job.sh.tpl | 25 +++++++++++++++++++ charts/mysqlclient-utility/Chart.yaml | 2 +- .../bin/utility/_mariadb_ondemand_job.sh.tpl | 25 +++++++++++++++++++ charts/postgresql-utility/Chart.yaml | 2 +- .../bin/utility/_pg_ondemand_job.sh.tpl | 25 +++++++++++++++++++ tools/helm_install.sh | 2 +- tools/helm_tk.sh | 2 +- 8 files changed, 80 insertions(+), 5 deletions(-) diff --git a/charts/etcdctl-utility/Chart.yaml b/charts/etcdctl-utility/Chart.yaml index 8cdf8cbb..f28bdaf2 100644 --- a/charts/etcdctl-utility/Chart.yaml +++ b/charts/etcdctl-utility/Chart.yaml @@ -13,4 +13,4 @@ apiVersion: v1 description: etcdctl Client name: etcdctl-utility -version: 0.1.4 +version: 0.1.5 diff --git a/charts/etcdctl-utility/templates/bin/utility/_etcd_ondemand_job.sh.tpl b/charts/etcdctl-utility/templates/bin/utility/_etcd_ondemand_job.sh.tpl index 58e45c95..55e5325a 100644 --- a/charts/etcdctl-utility/templates/bin/utility/_etcd_ondemand_job.sh.tpl +++ b/charts/etcdctl-utility/templates/bin/utility/_etcd_ondemand_job.sh.tpl @@ -117,6 +117,31 @@ spec: secretKeyRef: key: REMOTE_BACKUP_SEND_DELAY_MAX name: ${ETCD_CONF_SECRET} + - name: THROTTLE_BACKUPS_ENABLED + valueFrom: + secretKeyRef: + key: THROTTLE_BACKUPS_ENABLED + name: ${ETCD_CONF_SECRET} + - name: THROTTLE_LIMIT + valueFrom: + secretKeyRef: + key: THROTTLE_LIMIT + name: ${ETCD_CONF_SECRET} + - name: THROTTLE_LOCK_EXPIRE_AFTER + valueFrom: + secretKeyRef: + key: THROTTLE_LOCK_EXPIRE_AFTER + name: ${ETCD_CONF_SECRET} + - name: THROTTLE_RETRY_AFTER + valueFrom: + secretKeyRef: + key: THROTTLE_RETRY_AFTER + name: ${ETCD_CONF_SECRET} + - name: THROTTLE_CONTAINER_NAME + valueFrom: + secretKeyRef: + key: THROTTLE_CONTAINER_NAME + name: ${ETCD_CONF_SECRET} EOF if $ETCD_REMOTE_BACKUP_ENABLED; then diff --git a/charts/mysqlclient-utility/Chart.yaml b/charts/mysqlclient-utility/Chart.yaml index 7efdabaf..527463a3 100644 --- a/charts/mysqlclient-utility/Chart.yaml +++ b/charts/mysqlclient-utility/Chart.yaml @@ -13,4 +13,4 @@ apiVersion: v1 description: Porthole MySql Client name: mysqlclient-utility -version: 0.1.7 +version: 0.1.8 diff --git a/charts/mysqlclient-utility/templates/bin/utility/_mariadb_ondemand_job.sh.tpl b/charts/mysqlclient-utility/templates/bin/utility/_mariadb_ondemand_job.sh.tpl index 1fd8358d..afb843ca 100644 --- a/charts/mysqlclient-utility/templates/bin/utility/_mariadb_ondemand_job.sh.tpl +++ b/charts/mysqlclient-utility/templates/bin/utility/_mariadb_ondemand_job.sh.tpl @@ -146,6 +146,31 @@ spec: secretKeyRef: key: REMOTE_BACKUP_SEND_DELAY_MAX name: ${MARIADB_CONF_SECRET} + - name: THROTTLE_BACKUPS_ENABLED + valueFrom: + secretKeyRef: + key: THROTTLE_BACKUPS_ENABLED + name: ${MARIADB_CONF_SECRET} + - name: THROTTLE_LIMIT + valueFrom: + secretKeyRef: + key: THROTTLE_LIMIT + name: ${MARIADB_CONF_SECRET} + - name: THROTTLE_LOCK_EXPIRE_AFTER + valueFrom: + secretKeyRef: + key: THROTTLE_LOCK_EXPIRE_AFTER + name: ${MARIADB_CONF_SECRET} + - name: THROTTLE_RETRY_AFTER + valueFrom: + secretKeyRef: + key: THROTTLE_RETRY_AFTER + name: ${MARIADB_CONF_SECRET} + - name: THROTTLE_CONTAINER_NAME + valueFrom: + secretKeyRef: + key: THROTTLE_CONTAINER_NAME + name: ${MARIADB_CONF_SECRET} EOF if $MARIADB_REMOTE_BACKUP_ENABLED; then diff --git a/charts/postgresql-utility/Chart.yaml b/charts/postgresql-utility/Chart.yaml index 03b5cccd..e0bca318 100644 --- a/charts/postgresql-utility/Chart.yaml +++ b/charts/postgresql-utility/Chart.yaml @@ -14,4 +14,4 @@ apiVersion: v1 description: PostgreSQL Client name: postgresql-utility -version: 0.1.3 +version: 0.1.4 diff --git a/charts/postgresql-utility/templates/bin/utility/_pg_ondemand_job.sh.tpl b/charts/postgresql-utility/templates/bin/utility/_pg_ondemand_job.sh.tpl index 68de0e62..04fc8a06 100644 --- a/charts/postgresql-utility/templates/bin/utility/_pg_ondemand_job.sh.tpl +++ b/charts/postgresql-utility/templates/bin/utility/_pg_ondemand_job.sh.tpl @@ -130,6 +130,31 @@ spec: secretKeyRef: key: REMOTE_BACKUP_SEND_DELAY_MAX name: ${POSTGRESQL_CONF_SECRET} + - name: THROTTLE_BACKUPS_ENABLED + valueFrom: + secretKeyRef: + key: THROTTLE_BACKUPS_ENABLED + name: ${POSTGRESQL_CONF_SECRET} + - name: THROTTLE_LIMIT + valueFrom: + secretKeyRef: + key: THROTTLE_LIMIT + name: ${POSTGRESQL_CONF_SECRET} + - name: THROTTLE_LOCK_EXPIRE_AFTER + valueFrom: + secretKeyRef: + key: THROTTLE_LOCK_EXPIRE_AFTER + name: ${POSTGRESQL_CONF_SECRET} + - name: THROTTLE_RETRY_AFTER + valueFrom: + secretKeyRef: + key: THROTTLE_RETRY_AFTER + name: ${POSTGRESQL_CONF_SECRET} + - name: THROTTLE_CONTAINER_NAME + valueFrom: + secretKeyRef: + key: THROTTLE_CONTAINER_NAME + name: ${POSTGRESQL_CONF_SECRET} EOF if $POSTGRESQL_REMOTE_BACKUP_ENABLED; then diff --git a/tools/helm_install.sh b/tools/helm_install.sh index b3b1ca37..4060f051 100755 --- a/tools/helm_install.sh +++ b/tools/helm_install.sh @@ -17,7 +17,7 @@ set -x HELM=$1 -HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.12.2-linux-amd64.tar.gz"} +HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz"} function install_helm_binary { diff --git a/tools/helm_tk.sh b/tools/helm_tk.sh index 839a33d3..5757354c 100755 --- a/tools/helm_tk.sh +++ b/tools/helm_tk.sh @@ -17,7 +17,7 @@ set -eux HTK_REPO=${HTK_REPO:-"https://opendev.org/openstack/openstack-helm-infra.git"} -HTK_STABLE_COMMIT=${HTK_COMMIT:-"97ce6d7d8e9a090c748800d69a57bbd9af698b60"} +HTK_STABLE_COMMIT=${HTK_COMMIT:-"13c1d8cd3866bb026cce6e5d80555edfd0afa845"} TMP_DIR=$(mktemp -d)