openstack-helm/mariadb/templates/statefulset.yaml
Marcus d78f8e0901 Improve fault tolerance of MariaDB startup
* Changed podManagementPolicy to parallel in order to allow recovery
  from secondary or third master

  After rebooting the whole cluster on purpose or after a power failure
  a primary node the cluster can synchronize with is required. This is
  usually done automatically by selecting the node with the highest
  transaction id. The current implementation of the stateful set starts
  the nodes in sequence, preventing the start of further nodes if the
  process terminates with error state. Because of this, the cluster may
  not come up if the first or second node are not in primary state.

* Elects first node started in primary state as bootstrap source.
* Display warnings and runs mysqld with wsrep-recover on crashed nodes
* Introduces FORCE_RECOVERY argument for crash recovery

  In case the primary selection failed, the cluster bootstrap process
  must be manually initiated from the most advanced node (highest
  committed transaction id). This information is available from the
  grastate.dat file in case of a clean shutdown. On crashed nodes an
  InnoDB recovery is required to get the last committed transaction id.

  start.sh will handle both cases and gives instructions on how to
  recover the cluster on a hard failure. If FORCE_RECOVERY was set to
  the name of a POD (mariadb-0, mariadb-1, ...), the bootstrap process
  will be initiated from the specified node.

DocImpact
Closes-Bug: #1716461

Change-Id: I96a8cb52124f64920a7d9cf21a8924ede78ebf7b
2018-02-13 11:30:26 +00:00

164 lines
5.8 KiB
YAML

{{/*
Copyright 2017 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.statefulset }}
{{- $envAll := . }}
{{- $dependencies := .Values.dependencies.mariadb }}
{{- $serviceAccountName := "mariadb" }}
{{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
---
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
name: mariadb
spec:
serviceName: "{{ tuple "oslo_db" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}"
podManagementPolicy: "Parallel"
replicas: {{ .Values.pod.replicas.server }}
template:
metadata:
labels:
{{ tuple $envAll "mariadb" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
spec:
serviceAccountName: {{ $serviceAccountName }}
affinity:
{{ tuple $envAll "mariadb" "server" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }}
nodeSelector:
{{ .Values.labels.server.node_selector_key }}: {{ .Values.labels.server.node_selector_value }}
initContainers:
{{ tuple $envAll $dependencies list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
{{- if .Values.volume.chown_on_start }}
- name: mariadb-perms
image: {{ .Values.images.tags.mariadb }}
imagePullPolicy: {{ .Values.images.pull_policy }}
securityContext:
runAsUser: 0
{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
command:
- chown
- -R
- "mysql:mysql"
- /var/lib/mysql
volumeMounts:
- name: mysql-data
mountPath: /var/lib/mysql
{{- end }}
containers:
- name: mariadb
image: {{ .Values.images.tags.mariadb }}
imagePullPolicy: {{ .Values.images.pull_policy }}
{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
command:
- /tmp/start.sh
env:
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: FORCE_BOOTSTRAP
value: {{ .Values.force_bootstrap | quote }}
- name: FORCE_RECOVERY
value: {{ .Values.force_recovey | quote }}
- name: BOOTSTRAP_FILE
value: {{ printf "/tmp/%s.sql" (randAlphaNum 8) }}
- name: MARIADB_REPLICAS
value: {{ .Values.pod.replicas.server | quote }}
- name: WSREP_PORT
value: {{ tuple "oslo_db" "internal" "wsrep" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
- name: SERVICE_NAME
value: {{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
- name: MYSQL_ROOT_PASSWORD
valueFrom:
secretKeyRef:
name: mariadb-db-root-password
key: MYSQL_ROOT_PASSWORD
readinessProbe:
initialDelaySeconds: 30
periodSeconds: 30
timeoutSeconds: 3
exec:
command:
- /tmp/readiness.sh
volumeMounts:
- name: mycnfd
mountPath: /etc/mysql/conf.d
- name: mariadb-bin
mountPath: /tmp/readiness.sh
subPath: readiness.sh
readOnly: true
- name: mariadb-bin
mountPath: /tmp/start.sh
subPath: start.sh
readOnly: true
- name: mariadb-etc
mountPath: /etc/mysql/my.cnf
subPath: my.cnf
readOnly: true
- name: mariadb-etc
mountPath: /etc/mysql/conf.d/00-base.cnf
subPath: 00-base.cnf
readOnly: true
- name: mariadb-etc
mountPath: /etc/mysql/conf.d/20-override.cnf
subPath: 20-override.cnf
readOnly: true
- name: mariadb-etc
mountPath: /etc/mysql/conf.d/99-force.cnf
subPath: 99-force.cnf
readOnly: true
- name: mariadb-secrets
mountPath: /etc/mysql/admin_user.cnf
subPath: admin_user.cnf
readOnly: true
- name: mysql-data
mountPath: /var/lib/mysql
volumes:
- name: mycnfd
emptyDir: {}
- name: mariadb-bin
configMap:
name: mariadb-bin
defaultMode: 0555
- name: mariadb-etc
configMap:
name: mariadb-etc
defaultMode: 0444
- name: mariadb-secrets
secret:
secretName: mariadb-secrets
defaultMode: 0444
{{- if not .Values.volume.enabled }}
- name: mysql-data
emptyDir: {}
{{- end }}
{{- if .Values.volume.enabled }}
volumeClaimTemplates:
- metadata:
name: mysql-data
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: {{ .Values.volume.size }}
storageClassName: {{ .Values.volume.class_name }}
{{- end }}
{{- end }}