[backups] Added staggered backups

This PS adds staggered backups possibility by adding anti-affinity rules
to backups cronjobs that can be followed across several namespaces to
decrease load on remote backup destination server making sure that at
every moment in time there is only one backup upload is in progress.

Change-Id: If49791f866a73a08fb98fa0e0b4854042d079c66

helm-toolkit/Chart.yaml

@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v1.0.0
 description: OpenStack-Helm Helm-Toolkit
 name: helm-toolkit
-version: 0.2.56
+version: 0.2.57
 home: https://docs.openstack.org/openstack-helm
 icon: https://www.openstack.org/themes/openstack/images/project-mascots/OpenStack-Helm/OpenStack_Project_OpenStackHelm_vertical.png
 sources:

helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl

@@ -243,7 +243,7 @@ send_to_remote_server() {
       log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup MD5 hash values"
       return 2
   fi
-  rm -rf ${REMOTE_FILE}
+  rm -f ${REMOTE_FILE}
 
   log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully."
   return 0

mariadb-backup/Chart.yaml

@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v10.6.14
 description: OpenStack-Helm MariaDB backups
 name: mariadb-backup
-version: 0.0.1
+version: 0.0.2
 home: https://mariadb.com/kb/en/
 icon: http://badges.mariadb.org/mariadb-badge-180x60.png
 sources:

mariadb-backup/templates/cron-job-backup-mariadb.yaml

@@ -56,6 +56,12 @@ spec:
 {{ if $envAll.Values.pod.tolerations.mariadb.enabled }}
 {{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 10 }}
 {{ end }}
+{{- if $envAll.Values.pod.affinity }}
+{{- if $envAll.Values.pod.affinity.mariadb_backup }}
+          affinity:
+{{  index $envAll.Values.pod.affinity "mariadb_backup"  | toYaml | indent 12}}
+{{- end }}
+{{- end }}
           nodeSelector:
             {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
           initContainers:

mariadb-backup/values_overrides/staggered-backups.yaml

@@ -0,0 +1,38 @@
+---
+conf:
+  backup:
+    enabled: true
+    remote_backup:
+      enabled: false
+pod:
+  labels:
+    backup:
+      staggered_backups: enabled
+  affinity:
+    mariadb_backup:
+      podAntiAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+                - key: status.phase
+                  operator: NotIn
+                  values:
+                    - Running
+                - key: staggered-backups
+                  operator: In
+                  values:
+                    - enabled
+            namespaces:
+              - openstack
+              - osh-infra
+              - ucp
+            topologyKey: kubernetes.io/os
+volume:
+  backup:
+    enabled: true
+manifests:
+  pvc_backup: true
+  job_ks_user: false
+  cron_job_mariadb_backup: true
+  secret_backup_restore: true
+...

mariadb/Chart.yaml

@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v10.6.7
 description: OpenStack-Helm MariaDB
 name: mariadb
-version: 0.2.35
+version: 0.2.36
 home: https://mariadb.com/kb/en/
 icon: http://badges.mariadb.org/mariadb-badge-180x60.png
 sources:

mariadb/templates/cron-job-backup-mariadb.yaml

@@ -53,9 +53,15 @@ spec:
           serviceAccountName: {{ $serviceAccountName }}
           restartPolicy: OnFailure
           shareProcessNamespace: true
-{{ if $envAll.Values.pod.tolerations.mariadb.enabled }}
+{{- if $envAll.Values.pod.tolerations.mariadb.enabled }}
 {{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 10 }}
-{{ end }}
+{{- end }}
+{{- if $envAll.Values.pod.affinity }}
+{{- if $envAll.Values.pod.affinity.mariadb_backup }}
+          affinity:
+{{  index $envAll.Values.pod.affinity "mariadb_backup"  | toYaml | indent 12}}
+{{- end }}
+{{- end }}
           nodeSelector:
             {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
           initContainers:
@@ -191,9 +197,6 @@ spec:
                   mountPath: /tmp/start_verification_server.sh
                   readOnly: true
                   subPath: start_verification_server.sh
-          restartPolicy: OnFailure
-          serviceAccount: {{ $serviceAccountName }}
-          serviceAccountName: {{ $serviceAccountName }}
           volumes:
             - name: pod-tmp
               emptyDir: {}

mariadb/values_overrides/backups.yaml

@@ -0,0 +1,15 @@
+---
+conf:
+  backup:
+    enabled: true
+    remote_backup:
+      enabled: false
+volume:
+  backup:
+    enabled: true
+manifests:
+  pvc_backup: true
+  job_ks_user: false
+  cron_job_mariadb_backup: true
+  secret_backup_restore: true
+...

mariadb/values_overrides/staggered-backups.yaml

@@ -0,0 +1,38 @@
+---
+conf:
+  backup:
+    enabled: true
+    remote_backup:
+      enabled: false
+pod:
+  labels:
+    backup:
+      staggered_backups: enabled
+  affinity:
+    mariadb_backup:
+      podAntiAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+                - key: status.phase
+                  operator: NotIn
+                  values:
+                    - Running
+                - key: staggered-backups
+                  operator: In
+                  values:
+                    - enabled
+            namespaces:
+              - openstack
+              - osh-infra
+              - ucp
+            topologyKey: kubernetes.io/os
+volume:
+  backup:
+    enabled: true
+manifests:
+  pvc_backup: true
+  job_ks_user: false
+  cron_job_mariadb_backup: true
+  secret_backup_restore: true
+...

postgresql/Chart.yaml

@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v14.5
 description: OpenStack-Helm PostgreSQL
 name: postgresql
-version: 0.1.18
+version: 0.1.19
 home: https://www.postgresql.org
 sources:
   - https://github.com/postgres/postgres

postgresql/templates/cron-job-backup-postgres.yaml

@@ -52,6 +52,12 @@ spec:
 {{ dict "envAll" $envAll "application" "postgresql_backup" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 10 }}
           serviceAccountName: {{ $serviceAccountName }}
           restartPolicy: OnFailure
+{{- if $envAll.Values.pod.affinity }}
+{{- if $envAll.Values.pod.affinity.postgresql_backup }}
+          affinity:
+{{  index $envAll.Values.pod.affinity "postgresql_backup"  | toYaml | indent 12}}
+{{- end }}
+{{- end }}
           nodeSelector:
             {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
           initContainers:
@@ -137,9 +143,6 @@ spec:
                   mountPath: /etc/postgresql/admin_user.conf
                   subPath: admin_user.conf
                   readOnly: true
-          restartPolicy: OnFailure
-          serviceAccount: {{ $serviceAccountName }}
-          serviceAccountName: {{ $serviceAccountName }}
           volumes:
             - name: pod-tmp
               emptyDir: {}

postgresql/values_overrides/backups.yaml

@@ -0,0 +1,15 @@
+---
+conf:
+  backup:
+    enabled: true
+    remote_backup:
+      enabled: false
+volume:
+  backup:
+    enabled: true
+manifests:
+  pvc_backup: true
+  job_ks_user: false
+  cron_job_postgresql_backup: true
+  secret_backup_restore: true
+...

postgresql/values_overrides/staggered-backups.yaml

@@ -0,0 +1,38 @@
+---
+conf:
+  backup:
+    enabled: true
+    remote_backup:
+      enabled: false
+pod:
+  labels:
+    backup:
+      staggered_backups: enabled
+  affinity:
+    postgresql_backup:
+      podAntiAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+                - key: status.phase
+                  operator: NotIn
+                  values:
+                    - Running
+                - key: staggered-backups
+                  operator: In
+                  values:
+                    - enabled
+            namespaces:
+              - openstack
+              - osh-infra
+              - ucp
+            topologyKey: kubernetes.io/os
+volume:
+  backup:
+    enabled: true
+manifests:
+  pvc_backup: true
+  job_ks_user: false
+  cron_job_postgresql_backup: true
+  secret_backup_restore: true
+...

releasenotes/notes/helm-toolkit.yaml

@@ -63,4 +63,5 @@ helm-toolkit:
   - 0.2.54 Fix dependency resolver to ignore non-existing dependencyKey when dependencyMixinParam is a slice
   - 0.2.55 Updated deprecated IngressClass annotation
   - 0.2.56 Expose S3 credentials from Rook bucket CRD secret
+  - 0.2.57 Safer file removal
 ...

releasenotes/notes/mariadb-backup.yaml

@@ -1,4 +1,5 @@
 ---
 mariadb-backup:
   - 0.0.1 Initial Chart
+  - 0.0.2 Added staggered backups support
 ...

releasenotes/notes/mariadb.yaml

@@ -51,4 +51,5 @@ mariadb:
   - 0.2.33 Add 2023.1 Ubuntu Focal overrides
   - 0.2.34 Uplift ingress controller image to 1.8.2
   - 0.2.35 Update apparmor override
+  - 0.2.36 Added staggered backups support
 ...

releasenotes/notes/postgresql.yaml

@@ -19,4 +19,5 @@ postgresql:
   - 0.1.16 Added OCI registry authentication
   - 0.1.17 Added empty verify_databases_backup_archives() function implementation to match updated backup_databases() function in helm-toolkit
   - 0.1.18 Updated postgres to 14.5 and replaced deprecated config item wal_keep_segments with wal_keep_size
+  - 0.1.19 Added staggered backups support
 ...