From 4a95f75b6bbd84f889fd95056e67f7bdda3965b9 Mon Sep 17 00:00:00 2001 From: Sergiy Markin Date: Fri, 1 Dec 2023 22:09:13 +0000 Subject: [PATCH] [backups] Added staggered backups This PS adds staggered backups possibility by adding anti-affinity rules to backups cronjobs that can be followed across several namespaces to decrease load on remote backup destination server making sure that at every moment in time there is only one backup upload is in progress. Change-Id: If49791f866a73a08fb98fa0e0b4854042d079c66 --- helm-toolkit/Chart.yaml | 2 +- .../db-backup-restore/_backup_main.sh.tpl | 2 +- mariadb-backup/Chart.yaml | 2 +- .../templates/cron-job-backup-mariadb.yaml | 6 +++ .../values_overrides/staggered-backups.yaml | 38 +++++++++++++++++++ mariadb/Chart.yaml | 2 +- .../templates/cron-job-backup-mariadb.yaml | 13 ++++--- mariadb/values_overrides/backups.yaml | 15 ++++++++ .../values_overrides/staggered-backups.yaml | 38 +++++++++++++++++++ postgresql/Chart.yaml | 2 +- .../templates/cron-job-backup-postgres.yaml | 9 +++-- postgresql/values_overrides/backups.yaml | 15 ++++++++ .../values_overrides/staggered-backups.yaml | 38 +++++++++++++++++++ releasenotes/notes/helm-toolkit.yaml | 1 + releasenotes/notes/mariadb-backup.yaml | 1 + releasenotes/notes/mariadb.yaml | 1 + releasenotes/notes/postgresql.yaml | 1 + 17 files changed, 173 insertions(+), 13 deletions(-) create mode 100644 mariadb-backup/values_overrides/staggered-backups.yaml create mode 100644 mariadb/values_overrides/backups.yaml create mode 100644 mariadb/values_overrides/staggered-backups.yaml create mode 100644 postgresql/values_overrides/backups.yaml create mode 100644 postgresql/values_overrides/staggered-backups.yaml diff --git a/helm-toolkit/Chart.yaml b/helm-toolkit/Chart.yaml index e4b45e31c..a22b4f9c2 100644 --- a/helm-toolkit/Chart.yaml +++ b/helm-toolkit/Chart.yaml @@ -15,7 +15,7 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Helm-Toolkit name: helm-toolkit -version: 0.2.56 +version: 0.2.57 home: https://docs.openstack.org/openstack-helm icon: https://www.openstack.org/themes/openstack/images/project-mascots/OpenStack-Helm/OpenStack_Project_OpenStackHelm_vertical.png sources: diff --git a/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl b/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl index 3963bd405..2634b6da2 100755 --- a/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl +++ b/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl @@ -243,7 +243,7 @@ send_to_remote_server() { log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup MD5 hash values" return 2 fi - rm -rf ${REMOTE_FILE} + rm -f ${REMOTE_FILE} log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully." return 0 diff --git a/mariadb-backup/Chart.yaml b/mariadb-backup/Chart.yaml index a34b23fc2..f98d06d08 100644 --- a/mariadb-backup/Chart.yaml +++ b/mariadb-backup/Chart.yaml @@ -15,7 +15,7 @@ apiVersion: v1 appVersion: v10.6.14 description: OpenStack-Helm MariaDB backups name: mariadb-backup -version: 0.0.1 +version: 0.0.2 home: https://mariadb.com/kb/en/ icon: http://badges.mariadb.org/mariadb-badge-180x60.png sources: diff --git a/mariadb-backup/templates/cron-job-backup-mariadb.yaml b/mariadb-backup/templates/cron-job-backup-mariadb.yaml index 18dd3e0fd..87fd91c41 100644 --- a/mariadb-backup/templates/cron-job-backup-mariadb.yaml +++ b/mariadb-backup/templates/cron-job-backup-mariadb.yaml @@ -56,6 +56,12 @@ spec: {{ if $envAll.Values.pod.tolerations.mariadb.enabled }} {{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 10 }} {{ end }} +{{- if $envAll.Values.pod.affinity }} +{{- if $envAll.Values.pod.affinity.mariadb_backup }} + affinity: +{{ index $envAll.Values.pod.affinity "mariadb_backup" | toYaml | indent 12}} +{{- end }} +{{- end }} nodeSelector: {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} initContainers: diff --git a/mariadb-backup/values_overrides/staggered-backups.yaml b/mariadb-backup/values_overrides/staggered-backups.yaml new file mode 100644 index 000000000..03412d748 --- /dev/null +++ b/mariadb-backup/values_overrides/staggered-backups.yaml @@ -0,0 +1,38 @@ +--- +conf: + backup: + enabled: true + remote_backup: + enabled: false +pod: + labels: + backup: + staggered_backups: enabled + affinity: + mariadb_backup: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: status.phase + operator: NotIn + values: + - Running + - key: staggered-backups + operator: In + values: + - enabled + namespaces: + - openstack + - osh-infra + - ucp + topologyKey: kubernetes.io/os +volume: + backup: + enabled: true +manifests: + pvc_backup: true + job_ks_user: false + cron_job_mariadb_backup: true + secret_backup_restore: true +... diff --git a/mariadb/Chart.yaml b/mariadb/Chart.yaml index 101e14257..e85973474 100644 --- a/mariadb/Chart.yaml +++ b/mariadb/Chart.yaml @@ -15,7 +15,7 @@ apiVersion: v1 appVersion: v10.6.7 description: OpenStack-Helm MariaDB name: mariadb -version: 0.2.35 +version: 0.2.36 home: https://mariadb.com/kb/en/ icon: http://badges.mariadb.org/mariadb-badge-180x60.png sources: diff --git a/mariadb/templates/cron-job-backup-mariadb.yaml b/mariadb/templates/cron-job-backup-mariadb.yaml index db8c06639..e6974ef42 100644 --- a/mariadb/templates/cron-job-backup-mariadb.yaml +++ b/mariadb/templates/cron-job-backup-mariadb.yaml @@ -53,9 +53,15 @@ spec: serviceAccountName: {{ $serviceAccountName }} restartPolicy: OnFailure shareProcessNamespace: true -{{ if $envAll.Values.pod.tolerations.mariadb.enabled }} +{{- if $envAll.Values.pod.tolerations.mariadb.enabled }} {{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 10 }} -{{ end }} +{{- end }} +{{- if $envAll.Values.pod.affinity }} +{{- if $envAll.Values.pod.affinity.mariadb_backup }} + affinity: +{{ index $envAll.Values.pod.affinity "mariadb_backup" | toYaml | indent 12}} +{{- end }} +{{- end }} nodeSelector: {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} initContainers: @@ -191,9 +197,6 @@ spec: mountPath: /tmp/start_verification_server.sh readOnly: true subPath: start_verification_server.sh - restartPolicy: OnFailure - serviceAccount: {{ $serviceAccountName }} - serviceAccountName: {{ $serviceAccountName }} volumes: - name: pod-tmp emptyDir: {} diff --git a/mariadb/values_overrides/backups.yaml b/mariadb/values_overrides/backups.yaml new file mode 100644 index 000000000..5a7de206c --- /dev/null +++ b/mariadb/values_overrides/backups.yaml @@ -0,0 +1,15 @@ +--- +conf: + backup: + enabled: true + remote_backup: + enabled: false +volume: + backup: + enabled: true +manifests: + pvc_backup: true + job_ks_user: false + cron_job_mariadb_backup: true + secret_backup_restore: true +... diff --git a/mariadb/values_overrides/staggered-backups.yaml b/mariadb/values_overrides/staggered-backups.yaml new file mode 100644 index 000000000..03412d748 --- /dev/null +++ b/mariadb/values_overrides/staggered-backups.yaml @@ -0,0 +1,38 @@ +--- +conf: + backup: + enabled: true + remote_backup: + enabled: false +pod: + labels: + backup: + staggered_backups: enabled + affinity: + mariadb_backup: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: status.phase + operator: NotIn + values: + - Running + - key: staggered-backups + operator: In + values: + - enabled + namespaces: + - openstack + - osh-infra + - ucp + topologyKey: kubernetes.io/os +volume: + backup: + enabled: true +manifests: + pvc_backup: true + job_ks_user: false + cron_job_mariadb_backup: true + secret_backup_restore: true +... diff --git a/postgresql/Chart.yaml b/postgresql/Chart.yaml index b9ee4aa1c..a5443202b 100644 --- a/postgresql/Chart.yaml +++ b/postgresql/Chart.yaml @@ -15,7 +15,7 @@ apiVersion: v1 appVersion: v14.5 description: OpenStack-Helm PostgreSQL name: postgresql -version: 0.1.18 +version: 0.1.19 home: https://www.postgresql.org sources: - https://github.com/postgres/postgres diff --git a/postgresql/templates/cron-job-backup-postgres.yaml b/postgresql/templates/cron-job-backup-postgres.yaml index 3d9394d45..c2e2e8d26 100644 --- a/postgresql/templates/cron-job-backup-postgres.yaml +++ b/postgresql/templates/cron-job-backup-postgres.yaml @@ -52,6 +52,12 @@ spec: {{ dict "envAll" $envAll "application" "postgresql_backup" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 10 }} serviceAccountName: {{ $serviceAccountName }} restartPolicy: OnFailure +{{- if $envAll.Values.pod.affinity }} +{{- if $envAll.Values.pod.affinity.postgresql_backup }} + affinity: +{{ index $envAll.Values.pod.affinity "postgresql_backup" | toYaml | indent 12}} +{{- end }} +{{- end }} nodeSelector: {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} initContainers: @@ -137,9 +143,6 @@ spec: mountPath: /etc/postgresql/admin_user.conf subPath: admin_user.conf readOnly: true - restartPolicy: OnFailure - serviceAccount: {{ $serviceAccountName }} - serviceAccountName: {{ $serviceAccountName }} volumes: - name: pod-tmp emptyDir: {} diff --git a/postgresql/values_overrides/backups.yaml b/postgresql/values_overrides/backups.yaml new file mode 100644 index 000000000..499322a81 --- /dev/null +++ b/postgresql/values_overrides/backups.yaml @@ -0,0 +1,15 @@ +--- +conf: + backup: + enabled: true + remote_backup: + enabled: false +volume: + backup: + enabled: true +manifests: + pvc_backup: true + job_ks_user: false + cron_job_postgresql_backup: true + secret_backup_restore: true +... diff --git a/postgresql/values_overrides/staggered-backups.yaml b/postgresql/values_overrides/staggered-backups.yaml new file mode 100644 index 000000000..f51ba78c9 --- /dev/null +++ b/postgresql/values_overrides/staggered-backups.yaml @@ -0,0 +1,38 @@ +--- +conf: + backup: + enabled: true + remote_backup: + enabled: false +pod: + labels: + backup: + staggered_backups: enabled + affinity: + postgresql_backup: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: status.phase + operator: NotIn + values: + - Running + - key: staggered-backups + operator: In + values: + - enabled + namespaces: + - openstack + - osh-infra + - ucp + topologyKey: kubernetes.io/os +volume: + backup: + enabled: true +manifests: + pvc_backup: true + job_ks_user: false + cron_job_postgresql_backup: true + secret_backup_restore: true +... diff --git a/releasenotes/notes/helm-toolkit.yaml b/releasenotes/notes/helm-toolkit.yaml index 44e26149d..0df02e72b 100644 --- a/releasenotes/notes/helm-toolkit.yaml +++ b/releasenotes/notes/helm-toolkit.yaml @@ -63,4 +63,5 @@ helm-toolkit: - 0.2.54 Fix dependency resolver to ignore non-existing dependencyKey when dependencyMixinParam is a slice - 0.2.55 Updated deprecated IngressClass annotation - 0.2.56 Expose S3 credentials from Rook bucket CRD secret + - 0.2.57 Safer file removal ... diff --git a/releasenotes/notes/mariadb-backup.yaml b/releasenotes/notes/mariadb-backup.yaml index e8bc615c6..192fa1d39 100644 --- a/releasenotes/notes/mariadb-backup.yaml +++ b/releasenotes/notes/mariadb-backup.yaml @@ -1,4 +1,5 @@ --- mariadb-backup: - 0.0.1 Initial Chart + - 0.0.2 Added staggered backups support ... diff --git a/releasenotes/notes/mariadb.yaml b/releasenotes/notes/mariadb.yaml index d7ac44994..4f745ab91 100644 --- a/releasenotes/notes/mariadb.yaml +++ b/releasenotes/notes/mariadb.yaml @@ -51,4 +51,5 @@ mariadb: - 0.2.33 Add 2023.1 Ubuntu Focal overrides - 0.2.34 Uplift ingress controller image to 1.8.2 - 0.2.35 Update apparmor override + - 0.2.36 Added staggered backups support ... diff --git a/releasenotes/notes/postgresql.yaml b/releasenotes/notes/postgresql.yaml index 85bec52b4..9a8368448 100644 --- a/releasenotes/notes/postgresql.yaml +++ b/releasenotes/notes/postgresql.yaml @@ -19,4 +19,5 @@ postgresql: - 0.1.16 Added OCI registry authentication - 0.1.17 Added empty verify_databases_backup_archives() function implementation to match updated backup_databases() function in helm-toolkit - 0.1.18 Updated postgres to 14.5 and replaced deprecated config item wal_keep_segments with wal_keep_size + - 0.1.19 Added staggered backups support ...