openstack/openstack-helm-infra
Code Issues Proposed changes
adf9fc7fc4
openstack-helm-infra/ceph-client/templates/deployment-mgr.yaml
Frank Ritchie 5909bcbdef Use hostPID for ceph-mgr deployment 
This change is to address a memory leak in the ceph-mgr deployment.
The leak has also been noted in:

https://review.opendev.org/#/c/711085

Without this change memory usage for the active ceph-mgr pod will
steadily increase by roughly 100MiB per hour until all available
memory has been exhausted. Reset messages will also be seen in the
active and standby ceph-mgr pod logs.

Sample messages:

---

0 client.0 ms_handle_reset on v2:10.0.0.226:6808/1
0 client.0 ms_handle_reset on v2:10.0.0.226:6808/1
0 client.0 ms_handle_reset on v2:10.0.0.226:6808/1

---

The root cause of the resets and associated memory leak appears to
be due to multiple ceph pods sharing the same IP address (due to
hostNetwork being true) and PID (due to hostPID being false).
In the messages above the "1" at the end of the line is the PID.
Ceph appears to use the Version:IP:Port/PID (v2:10.0.0.226:6808/1)
tuple as a unique identifier. When hostPID is false conflicts arise.

Setting hostPID to true stops the reset messages and memory leak.

Change-Id: I9821637e75e8f89b59cf39842a6eb7e66518fa2c
2020-08-03 17:35:51 +00:00

202 lines
8.1 KiB
YAML
Raw Blame History

{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if and .Values.manifests.deployment_mgr (and .Values.deployment.ceph .Values.conf.features.mgr ) }}
{{- $envAll := . }}
{{- $serviceAccountName := "ceph-mgr" }}
{{ tuple $envAll "mgr" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
---
kind: Deployment
apiVersion: apps/v1
metadata:
name: ceph-mgr
annotations:
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
labels:
{{ tuple $envAll "ceph" "mgr" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
spec:
replicas: {{ .Values.pod.replicas.mgr }}
selector:
matchLabels:
{{ tuple $envAll "ceph" "mgr" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }}
strategy:
type: {{ .Values.pod.updateStrategy.mgr.type }}
template:
metadata:
labels:
{{ tuple $envAll "ceph" "mgr" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
annotations:
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
configmap-etc-client-hash: {{ tuple "configmap-etc-client.yaml" . | include "helm-toolkit.utils.hash" }}
{{ dict "envAll" $envAll "podName" "ceph-mgr" "containerNames" (list "ceph-mgr" "ceph-init-dirs") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
spec:
{{ dict "envAll" $envAll "application" "mgr" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
serviceAccountName: {{ $serviceAccountName }}
affinity:
{{ tuple $envAll "ceph" "mgr" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }}
{{ tuple $envAll "mgr" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }}
nodeSelector:
{{ .Values.labels.mgr.node_selector_key }}: {{ .Values.labels.mgr.node_selector_value }}
hostNetwork: true
hostPID: true
dnsPolicy: {{ .Values.pod.dns_policy }}
initContainers:
{{ tuple $envAll "mgr" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
- name: ceph-init-dirs
{{ tuple $envAll "ceph_mds" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ dict "envAll" $envAll "application" "mgr" "container" "init_dirs" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
command:
- /tmp/init-dirs.sh
env:
- name: CLUSTER
value: "ceph"
volumeMounts:
- name: pod-tmp
mountPath: /tmp
- name: pod-run
mountPath: /run
- name: pod-etc-ceph
mountPath: /etc/ceph
- name: ceph-client-bin
mountPath: /tmp/init-dirs.sh
subPath: init-dirs.sh
readOnly: true
- name: pod-var-lib-ceph
mountPath: /var/lib/ceph
readOnly: false
containers:
- name: ceph-mgr
{{ tuple $envAll "ceph_mgr" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.mgr | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
{{ dict "envAll" $envAll "application" "mgr" "container" "mgr" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
env:
- name: CLUSTER
value: "ceph"
- name: NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: MON_PORT
value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
- name: MON_PORT_V2
value: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
{{- if .Values.ceph_mgr_enabled_modules }}
- name: ENABLED_MODULES
value: |-
{{- range $value := .Values.ceph_mgr_enabled_modules }}
{{ $value }}
{{- end }}
{{- end }}
{{- if .Values.ceph_mgr_modules_config }}
{{- range $module,$params := .Values.ceph_mgr_modules_config }}
{{- range $key, $value := $params }}
- name: {{ $module }}_{{ $key }}
value: {{ $value | quote }}
{{- end }}
{{- end }}
{{- end }}
command:
- /mgr-start.sh
ports:
- name: mgr
containerPort: {{ tuple "ceph_mgr" "internal" "mgr" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
{{- if (has "prometheus" .Values.ceph_mgr_enabled_modules) }}
- name: metrics
containerPort: {{ tuple "ceph_mgr" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
{{ end -}}
livenessProbe:
exec:
command:
- /tmp/mgr-check.sh
- liveness
initialDelaySeconds: 30
timeoutSeconds: 5
readinessProbe:
exec:
command:
- /tmp/mgr-check.sh
- readiness
initialDelaySeconds: 30
timeoutSeconds: 5
volumeMounts:
- name: pod-tmp
mountPath: /tmp
- name: pod-run
mountPath: /run
- name: pod-etc-ceph
mountPath: /etc/ceph
- name: ceph-client-bin
mountPath: /mgr-start.sh
subPath: mgr-start.sh
readOnly: true
- name: ceph-client-bin
mountPath: /tmp/mgr-check.sh
subPath: mgr-check.sh
readOnly: true
- name: ceph-client-bin
mountPath: /tmp/utils-checkDNS.sh
subPath: utils-checkDNS.sh
readOnly: true
- name: ceph-client-etc
mountPath: /etc/ceph/ceph.conf.template
subPath: ceph.conf
readOnly: true
- name: ceph-client-admin-keyring
mountPath: /etc/ceph/ceph.client.admin.keyring
subPath: ceph.client.admin.keyring
readOnly: true
- name: ceph-bootstrap-mgr-keyring
mountPath: /var/lib/ceph/bootstrap-mgr/ceph.keyring
subPath: ceph.keyring
readOnly: false
- name: pod-var-lib-ceph
mountPath: /var/lib/ceph
readOnly: false
- name: ceph-client-bin
mountPath: /tmp/utils-checkPGs.py
subPath: utils-checkPGs.py
readOnly: true
- name: ceph-client-bin
mountPath: /tmp/utils-checkPGs.sh
subPath: utils-checkPGs.sh
readOnly: true
volumes:
- name: pod-tmp
emptyDir: {}
- name: pod-run
emptyDir:
medium: "Memory"
- name: pod-etc-ceph
emptyDir: {}
- name: ceph-client-bin
configMap:
name: ceph-client-bin
defaultMode: 0555
- name: ceph-client-etc
configMap:
name: ceph-client-etc
defaultMode: 0444
- name: pod-var-lib-ceph
emptyDir: {}
- name: ceph-client-admin-keyring
secret:
secretName: {{ .Values.secrets.keyrings.admin }}
- name: ceph-bootstrap-mgr-keyring
secret:
secretName: {{ .Values.secrets.keyrings.mgr }}
{{- end }}
View Git Blame Copy Permalink