From 443ff3e3e340c94c5cbb214d1e2a8b2a3937541d Mon Sep 17 00:00:00 2001 From: Stephen Taylor Date: Tue, 15 Aug 2023 12:55:26 -0600 Subject: [PATCH] [ceph] Use Helm toolkit functions for Ceph probes This change converts the readiness and liveness probes in the Ceph charts to use the functions from the Helm toolkit rather than having hard-coded probe definitions. This allows probe configs to be overridden in values.yaml without rebuilding charts. Change-Id: I68a01b518f12d33fe4f87f86494a5f4e19be982e --- ceph-client/Chart.yaml | 2 +- ceph-client/templates/deployment-mds.yaml | 21 +++++++++-------- ceph-client/values.yaml | 12 ++++++++++ ceph-mon/Chart.yaml | 2 +- ceph-mon/templates/daemonset-mon.yaml | 28 +++++++++++------------ ceph-mon/templates/deployment-mgr.yaml | 28 +++++++++++------------ ceph-mon/values.yaml | 26 +++++++++++++++++++++ ceph-osd/Chart.yaml | 2 +- ceph-osd/templates/daemonset-osd.yaml | 28 +++++++++++------------ ceph-osd/values.yaml | 15 ++++++++++++ releasenotes/notes/ceph-client.yaml | 1 + releasenotes/notes/ceph-mon.yaml | 1 + releasenotes/notes/ceph-osd.yaml | 1 + 13 files changed, 113 insertions(+), 54 deletions(-) diff --git a/ceph-client/Chart.yaml b/ceph-client/Chart.yaml index 4f48d2bc7..4e669d732 100644 --- a/ceph-client/Chart.yaml +++ b/ceph-client/Chart.yaml @@ -15,6 +15,6 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Ceph Client name: ceph-client -version: 0.1.46 +version: 0.1.47 home: https://github.com/ceph/ceph-client ... diff --git a/ceph-client/templates/deployment-mds.yaml b/ceph-client/templates/deployment-mds.yaml index 2640c1c3d..ba67a8d47 100644 --- a/ceph-client/templates/deployment-mds.yaml +++ b/ceph-client/templates/deployment-mds.yaml @@ -12,6 +12,16 @@ See the License for the specific language governing permissions and limitations under the License. */}} +{{- define "livenessProbeTemplate" }} +tcpSocket: + port: 6800 +{{- end }} + +{{- define "readinessProbeTemplate" }} +tcpSocket: + port: 6800 +{{- end }} + {{- if and .Values.manifests.deployment_mds ( and .Values.deployment.ceph .Values.conf.features.mds) }} {{- $envAll := . }} @@ -100,15 +110,8 @@ spec: value: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} ports: - containerPort: 6800 - livenessProbe: - tcpSocket: - port: 6800 - initialDelaySeconds: 60 - timeoutSeconds: 5 - readinessProbe: - tcpSocket: - port: 6800 - timeoutSeconds: 5 +{{ dict "envAll" . "component" "ceph" "container" "ceph-mds" "type" "liveness" "probeTemplate" (include "livenessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | trim | indent 10 }} +{{ dict "envAll" . "component" "ceph" "container" "ceph-mds" "type" "readiness" "probeTemplate" (include "readinessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | trim | indent 10 }} volumeMounts: - name: pod-tmp mountPath: /tmp diff --git a/ceph-client/values.yaml b/ceph-client/values.yaml index 04d83bec8..0162ed2c9 100644 --- a/ceph-client/values.yaml +++ b/ceph-client/values.yaml @@ -179,6 +179,18 @@ pod: key: node.kubernetes.io/unreachable operator: Exists tolerationSeconds: 60 + probes: + ceph: + ceph-mds: + readiness: + enabled: true + params: + timeoutSeconds: 5 + liveness: + enabled: true + params: + initialDelaySeconds: 60 + timeoutSeconds: 5 secrets: keyrings: diff --git a/ceph-mon/Chart.yaml b/ceph-mon/Chart.yaml index f4ea83305..4294a495b 100644 --- a/ceph-mon/Chart.yaml +++ b/ceph-mon/Chart.yaml @@ -15,6 +15,6 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Ceph Mon name: ceph-mon -version: 0.1.29 +version: 0.1.30 home: https://github.com/ceph/ceph ... diff --git a/ceph-mon/templates/daemonset-mon.yaml b/ceph-mon/templates/daemonset-mon.yaml index a7368be01..1b6e9c933 100644 --- a/ceph-mon/templates/daemonset-mon.yaml +++ b/ceph-mon/templates/daemonset-mon.yaml @@ -12,6 +12,18 @@ See the License for the specific language governing permissions and limitations under the License. */}} +{{- define "monLivenessProbeTemplate" -}} +exec: + command: + - /tmp/mon-check.sh +{{- end -}} + +{{- define "monReadinessProbeTemplate" -}} +exec: + command: + - /tmp/mon-check.sh +{{- end -}} + {{- if and .Values.manifests.daemonset_mon .Values.deployment.ceph }} {{- $envAll := . }} @@ -175,20 +187,8 @@ spec: ports: - containerPort: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} - containerPort: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} - livenessProbe: - exec: - command: - - /tmp/mon-check.sh - - liveness - initialDelaySeconds: 360 - periodSeconds: 180 - readinessProbe: - exec: - command: - - /tmp/mon-check.sh - - readiness - initialDelaySeconds: 60 - periodSeconds: 60 +{{ dict "envAll" . "component" "ceph" "container" "ceph-mon" "type" "liveness" "probeTemplate" (include "monLivenessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | trim | indent 10 }} +{{ dict "envAll" . "component" "ceph" "container" "ceph-mon" "type" "readiness" "probeTemplate" (include "monReadinessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | trim | indent 10 }} volumeMounts: - name: pod-tmp mountPath: /tmp diff --git a/ceph-mon/templates/deployment-mgr.yaml b/ceph-mon/templates/deployment-mgr.yaml index b544276f7..7f2b4b123 100644 --- a/ceph-mon/templates/deployment-mgr.yaml +++ b/ceph-mon/templates/deployment-mgr.yaml @@ -12,6 +12,18 @@ See the License for the specific language governing permissions and limitations under the License. */}} +{{- define "mgrLivenessProbeTemplate" -}} +exec: + command: + - /tmp/mgr-check.sh +{{- end }} + +{{- define "mgrReadinessProbeTemplate" -}} +exec: + command: + - /tmp/mgr-check.sh +{{- end }} + {{- if and .Values.manifests.deployment_mgr (and .Values.deployment.ceph .Values.conf.features.mgr ) }} {{- $envAll := . }} @@ -126,20 +138,6 @@ spec: - name: metrics containerPort: {{ tuple "ceph_mgr" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} {{ end -}} - livenessProbe: - exec: - command: - - /tmp/mgr-check.sh - - liveness - initialDelaySeconds: 30 - timeoutSeconds: 5 - readinessProbe: - exec: - command: - - /tmp/mgr-check.sh - - readiness - initialDelaySeconds: 30 - timeoutSeconds: 5 volumeMounts: - name: pod-tmp mountPath: /tmp @@ -177,6 +175,8 @@ spec: mountPath: /tmp/utils-checkPGs.py subPath: utils-checkPGs.py readOnly: true +{{ dict "envAll" . "component" "ceph" "container" "ceph-mgr" "type" "liveness" "probeTemplate" (include "mgrLivenessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | trim | indent 10 }} +{{ dict "envAll" . "component" "ceph" "container" "ceph-mgr" "type" "readiness" "probeTemplate" (include "mgrReadinessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | trim | indent 10 }} volumes: - name: pod-tmp emptyDir: {} diff --git a/ceph-mon/values.yaml b/ceph-mon/values.yaml index 32e86ca75..c485c115f 100644 --- a/ceph-mon/values.yaml +++ b/ceph-mon/values.yaml @@ -207,6 +207,32 @@ pod: key: node.kubernetes.io/unreachable operator: Exists tolerationSeconds: 60 + probes: + ceph: + ceph-mon: + readiness: + enabled: true + params: + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 5 + liveness: + enabled: true + params: + initialDelaySeconds: 360 + periodSeconds: 180 + timeoutSeconds: 5 + ceph-mgr: + readiness: + enabled: true + params: + initialDelaySeconds: 30 + timeoutSeconds: 5 + liveness: + enabled: true + params: + initialDelaySeconds: 30 + timeoutSeconds: 5 secrets: keyrings: diff --git a/ceph-osd/Chart.yaml b/ceph-osd/Chart.yaml index d10448c15..f5a617da4 100644 --- a/ceph-osd/Chart.yaml +++ b/ceph-osd/Chart.yaml @@ -15,6 +15,6 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Ceph OSD name: ceph-osd -version: 0.1.45 +version: 0.1.46 home: https://github.com/ceph/ceph ... diff --git a/ceph-osd/templates/daemonset-osd.yaml b/ceph-osd/templates/daemonset-osd.yaml index 522f9e60f..3ba2ce7e9 100644 --- a/ceph-osd/templates/daemonset-osd.yaml +++ b/ceph-osd/templates/daemonset-osd.yaml @@ -12,6 +12,18 @@ See the License for the specific language governing permissions and limitations under the License. */}} +{{- define "osdLivenessProbeTemplate" -}} +exec: + command: + - /tmp/osd-check.sh +{{- end -}} + +{{- define "osdReadinessProbeTemplate" -}} +exec: + command: + - /tmp/osd-check.sh +{{- end -}} + {{- if .Values.manifests.daemonset_osd }} {{- $envAll := . }} @@ -352,20 +364,8 @@ spec: exec: command: - /tmp/osd-stop.sh - livenessProbe: - exec: - command: - - /tmp/osd-check.sh - - liveness - initialDelaySeconds: 120 - periodSeconds: 60 - readinessProbe: - exec: - command: - - /tmp/osd-check.sh - - readiness - initialDelaySeconds: 60 - periodSeconds: 60 +{{ dict "envAll" . "component" "ceph-osd" "container" "ceph-osd" "type" "liveness" "probeTemplate" (include "osdLivenessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | trim | indent 10 }} +{{ dict "envAll" . "component" "ceph-osd" "container" "ceph-osd" "type" "readiness" "probeTemplate" (include "osdReadinessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | trim | indent 10 }} volumeMounts: - name: pod-tmp mountPath: /tmp diff --git a/ceph-osd/values.yaml b/ceph-osd/values.yaml index 7fe7770d5..3179f3a37 100644 --- a/ceph-osd/values.yaml +++ b/ceph-osd/values.yaml @@ -137,6 +137,21 @@ pod: limits: memory: "1024Mi" cpu: "2000m" + probes: + ceph-osd: + ceph-osd: + readiness: + enabled: true + params: + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 5 + liveness: + enabled: true + params: + initialDelaySeconds: 120 + periodSeconds: 60 + timeoutSeconds: 5 secrets: keyrings: diff --git a/releasenotes/notes/ceph-client.yaml b/releasenotes/notes/ceph-client.yaml index 26da7f164..bddbe9dfe 100644 --- a/releasenotes/notes/ceph-client.yaml +++ b/releasenotes/notes/ceph-client.yaml @@ -47,4 +47,5 @@ ceph-client: - 0.1.44 Allow pg_num_min to be overridden per pool - 0.1.45 Update Ceph to 17.2.6 - 0.1.46 Strip any errors preceding pool properties JSON + - 0.1.47 Use Helm toolkit functions for Ceph probes ... diff --git a/releasenotes/notes/ceph-mon.yaml b/releasenotes/notes/ceph-mon.yaml index 4c7f32702..e8d4d6699 100644 --- a/releasenotes/notes/ceph-mon.yaml +++ b/releasenotes/notes/ceph-mon.yaml @@ -30,4 +30,5 @@ ceph-mon: - 0.1.27 Update all Ceph images to Focal - 0.1.28 Document the use of mon_allow_pool_size_one - 0.1.29 Update Ceph to 17.2.6 + - 0.1.30 Use Helm tookkit functions for Ceph probes ... diff --git a/releasenotes/notes/ceph-osd.yaml b/releasenotes/notes/ceph-osd.yaml index e4c50b4ed..a4c5fe6b6 100644 --- a/releasenotes/notes/ceph-osd.yaml +++ b/releasenotes/notes/ceph-osd.yaml @@ -46,4 +46,5 @@ ceph-osd: - 0.1.43 Update all Ceph images to Focal - 0.1.44 Update Ceph to 17.2.6 - 0.1.45 Extend the ceph-osd post-apply job PG wait + - 0.1.46 Use Helm toolkit functions for Ceph probes ...