Prevent liveness probe from killing mariadb pods during SST

Update liveness probe script to accept pods either sending
or receiving a SST, and avoid killing them.

Change-Id: I4ad95c45a7ab7e5e1cec2b4696671b6055cc10e7

mariadb/Chart.yaml

@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v10.6.7
 description: OpenStack-Helm MariaDB
 name: mariadb
-version: 0.2.31
+version: 0.2.32
 home: https://mariadb.com/kb/en/
 icon: http://badges.mariadb.org/mariadb-badge-180x60.png
 sources:

mariadb/templates/bin/_liveness.sh.tpl

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -e
+
+MYSQL="mysql \
+  --defaults-file=/etc/mysql/admin_user.cnf \
+  --host=localhost \
+{{- if .Values.manifests.certificates }}
+  --ssl-verify-server-cert=false \
+  --ssl-ca=/etc/mysql/certs/ca.crt \
+  --ssl-key=/etc/mysql/certs/tls.key \
+  --ssl-cert=/etc/mysql/certs/tls.crt \
+{{- end }}
+  --connect-timeout 2"
+
+mysql_status_query () {
+  STATUS=$1
+  $MYSQL -e "show status like \"${STATUS}\"" | \
+    awk "/${STATUS}/ { print \$NF; exit }"
+}
+
+{{- if eq (int .Values.pod.replicas.server) 1 }}
+if ! $MYSQL -e 'select 1' > /dev/null 2>&1 ; then
+  exit 1
+fi
+
+{{- else }}
+if [ -f /var/lib/mysql/sst_in_progress ]; then
+  # SST in progress, with this node receiving a snapshot.
+  # MariaDB won't be up yet; avoid killing.
+  exit 0
+fi
+
+if [ "x$(mysql_status_query wsrep_ready)" != "xON" ]; then
+  # WSREP says the node can receive queries
+  exit 1
+fi
+
+if [ "x$(mysql_status_query wsrep_connected)" != "xON" ]; then
+  # WSREP connected
+  exit 1
+fi
+
+if [ "x$(mysql_status_query wsrep_cluster_status)" != "xPrimary" ]; then
+  # Not in primary cluster
+  exit 1
+fi
+
+wsrep_local_state_comment=$(mysql_status_query wsrep_local_state_comment)
+if [ "x${wsrep_local_state_comment}" != "xSynced" ] && [ "x${wsrep_local_state_comment}" != "xDonor/Desynced" ]; then
+  # WSREP not synced or not sending SST
+  exit 1
+fi
+{{- end }}

mariadb/templates/configmap-bin.yaml

@@ -29,6 +29,8 @@ data:
 {{- end }}
   readiness.sh: |
 {{ tuple "bin/_readiness.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  liveness.sh: |
+{{ tuple "bin/_liveness.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   start.py: |
 {{ tuple "bin/_start.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   stop.sh: |

mariadb/templates/statefulset.yaml

@@ -17,6 +17,11 @@ exec:
   command:
     - /tmp/readiness.sh
 {{- end }}
+{{- define "mariadbLivenessProbe" }}
+exec:
+  command:
+    - /tmp/liveness.sh
+{{- end }}
 
 {{- if (.Values.global).subchart_release_name }}
 {{- $_ := set . "deployment_name" .Chart.Name }}
@@ -200,7 +205,7 @@ spec:
                 command:
                   - /tmp/stop.sh
 {{ dict "envAll" . "component" "server" "container" "mariadb" "type" "readiness" "probeTemplate" (include "mariadbReadinessProbe" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
-{{ dict "envAll" . "component" "server" "container" "mariadb" "type" "liveness" "probeTemplate" (include "mariadbReadinessProbe" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
+{{ dict "envAll" . "component" "server" "container" "mariadb" "type" "liveness" "probeTemplate" (include "mariadbLivenessProbe" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
           volumeMounts:
             - name: pod-tmp
               mountPath: /tmp
@@ -220,6 +225,10 @@ spec:
               mountPath: /tmp/readiness.sh
               subPath: readiness.sh
               readOnly: true
+            - name: mariadb-bin
+              mountPath: /tmp/liveness.sh
+              subPath: liveness.sh
+              readOnly: true
             - name: mariadb-etc
               mountPath: /etc/mysql/my.cnf
               subPath: my.cnf

releasenotes/notes/mariadb.yaml

@@ -47,4 +47,5 @@ mariadb:
   - 0.2.29 Uplift Mariadb-ingress to 1.5.1
   - 0.2.30 Replace node-role.kubernetes.io/master with control-plane
   - 0.2.31 Update kubernetes registry to registry.k8s.io
+  - 0.2.32 Prevent liveness probe from killing pods during SST
 ...