From ea2c0115c4c0d38e7f52037860017241792a87b7 Mon Sep 17 00:00:00 2001 From: Stephen Taylor Date: Wed, 2 Feb 2022 13:03:58 -0700 Subject: [PATCH] Move ceph-mgr deployment to the ceph-mon chart This change moves the ceph-mgr deployment from the ceph-client chart to the ceph-mon chart. Its purpose is to facilitate the proper Ceph upgrade procedure, which prescribes restarting mgr daemons before mon daemons. There will be additional work required to implement the correct daemon restart procedure for upgrades. This change only addresses the move of the ceph-mgr deployment. Change-Id: I3ac4a75f776760425c88a0ba1edae5fb339f128d --- ceph-client/Chart.yaml | 2 +- ceph-client/templates/configmap-bin.yaml | 7 -- ceph-client/templates/cronjob-checkPGs.yaml | 4 - ceph-client/values.yaml | 88 +---------------- ceph-client/values_overrides/apparmor.yaml | 3 - ceph-mon/Chart.yaml | 2 +- .../templates/bin/mgr/_check.sh.tpl | 0 .../templates/bin/mgr/_start.sh.tpl | 2 +- .../templates/bin/utils/_checkPGs.py.tpl | 0 ceph-mon/templates/configmap-bin.yaml | 8 ++ .../templates/deployment-mgr.yaml | 34 +++---- .../templates/service-mgr.yaml | 0 .../snippets/_mon_host_from_k8s_ep.sh.tpl | 68 +++++++++++++ ceph-mon/values.yaml | 99 +++++++++++++++++++ ceph-mon/values_overrides/apparmor.yaml | 3 + releasenotes/notes/ceph-client.yaml | 1 + releasenotes/notes/ceph-mon.yaml | 1 + 17 files changed, 199 insertions(+), 123 deletions(-) rename {ceph-client => ceph-mon}/templates/bin/mgr/_check.sh.tpl (100%) rename {ceph-client => ceph-mon}/templates/bin/mgr/_start.sh.tpl (97%) rename {ceph-client => ceph-mon}/templates/bin/utils/_checkPGs.py.tpl (100%) rename {ceph-client => ceph-mon}/templates/deployment-mgr.yaml (89%) rename {ceph-client => ceph-mon}/templates/service-mgr.yaml (100%) create mode 100644 ceph-mon/templates/snippets/_mon_host_from_k8s_ep.sh.tpl diff --git a/ceph-client/Chart.yaml b/ceph-client/Chart.yaml index 94a7a4a50..eb69de660 100644 --- a/ceph-client/Chart.yaml +++ b/ceph-client/Chart.yaml @@ -15,6 +15,6 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Ceph Client name: ceph-client -version: 0.1.29 +version: 0.1.30 home: https://github.com/ceph/ceph-client ... diff --git a/ceph-client/templates/configmap-bin.yaml b/ceph-client/templates/configmap-bin.yaml index 6caba70b3..627947353 100644 --- a/ceph-client/templates/configmap-bin.yaml +++ b/ceph-client/templates/configmap-bin.yaml @@ -43,11 +43,6 @@ data: mds-start.sh: | {{ tuple "bin/mds/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - mgr-start.sh: | -{{ tuple "bin/mgr/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - mgr-check.sh: | -{{ tuple "bin/mgr/_check.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - helm-tests.sh: | {{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} utils-checkDNS.sh: | @@ -55,8 +50,6 @@ data: utils-checkDNS_start.sh: | {{ tuple "bin/utils/_checkDNS_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - utils-checkPGs.py: | -{{ tuple "bin/utils/_checkPGs.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} utils-checkPGs.sh: | {{ tuple "bin/utils/_checkPGs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/ceph-client/templates/cronjob-checkPGs.yaml b/ceph-client/templates/cronjob-checkPGs.yaml index dca1488df..1d1cc2d91 100644 --- a/ceph-client/templates/cronjob-checkPGs.yaml +++ b/ceph-client/templates/cronjob-checkPGs.yaml @@ -91,10 +91,6 @@ spec: mountPath: /tmp - name: pod-etc-ceph mountPath: /etc/ceph - - name: ceph-client-bin - mountPath: /tmp/utils-checkPGs.py - subPath: utils-checkPGs.py - readOnly: true - name: ceph-client-bin mountPath: /tmp/utils-checkPGs.sh subPath: utils-checkPGs.sh diff --git a/ceph-client/values.yaml b/ceph-client/values.yaml index 38bc5e204..8cc40d0aa 100644 --- a/ceph-client/values.yaml +++ b/ceph-client/values.yaml @@ -27,7 +27,6 @@ images: ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:change_770201_ubuntu_bionic-20210113' ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:change_770201_ubuntu_bionic-20210113' ceph_mds: 'docker.io/openstackhelm/ceph-daemon:change_770201_ubuntu_bionic-20210113' - ceph_mgr: 'docker.io/openstackhelm/ceph-daemon:change_770201_ubuntu_bionic-20210113' ceph_rbd_pool: 'docker.io/openstackhelm/ceph-config-helper:change_770201_ubuntu_bionic-20210113' dep_check: 'quay.io/airshipit/kubernetes-entrypoint:v1.0.0' image_repo_sync: 'docker.io/library/docker:17.07.0' @@ -44,12 +43,12 @@ labels: test: node_selector_key: openstack-control-plane node_selector_value: enabled - mds: - node_selector_key: ceph-mds - node_selector_value: enabled mgr: node_selector_key: ceph-mgr node_selector_value: enabled + mds: + node_selector_key: ceph-mds + node_selector_value: enabled checkdns: node_selector_key: ceph-mon node_selector_value: enabled @@ -74,17 +73,6 @@ pod: runAsUser: 64045 readOnlyRootFilesystem: true allowPrivilegeEscalation: false - mgr: - pod: - runAsUser: 65534 - container: - init_dirs: - runAsUser: 0 - readOnlyRootFilesystem: true - mgr: - runAsUser: 64045 - readOnlyRootFilesystem: true - allowPrivilegeEscalation: false bootstrap: pod: runAsUser: 65534 @@ -109,7 +97,6 @@ pod: dns_policy: "ClusterFirstWithHostNet" replicas: mds: 2 - mgr: 2 lifecycle: upgrades: deployments: @@ -118,9 +105,6 @@ pod: rolling_update: max_surge: 25% max_unavailable: 25% - updateStrategy: - mgr: - type: Recreate affinity: anti: type: @@ -138,13 +122,6 @@ pod: limits: memory: "50Mi" cpu: "500m" - mgr: - requests: - memory: "5Mi" - cpu: "250m" - limits: - memory: "50Mi" - cpu: "500m" checkdns: requests: memory: "5Mi" @@ -202,16 +179,6 @@ pod: key: node.kubernetes.io/unreachable operator: Exists tolerationSeconds: 60 - mgr: - tolerations: - - effect: NoExecute - key: node.kubernetes.io/not-ready - operator: Exists - tolerationSeconds: 60 - - effect: NoExecute - key: node.kubernetes.io/unreachable - operator: Exists - tolerationSeconds: 60 secrets: keyrings: @@ -259,7 +226,6 @@ jobs: conf: features: mds: true - mgr: true pg_autoscaler: true cluster_flags: # List of flags to set or unset separated by spaces @@ -489,13 +455,6 @@ dependencies: services: - endpoint: internal service: ceph_mon - mgr: - jobs: - - ceph-storage-keys-generator - - ceph-mgr-keyring-generator - services: - - endpoint: internal - service: ceph_mon pool_checkpgs: jobs: - ceph-rbd-pool @@ -542,38 +501,6 @@ bootstrap: } #ensure_pool volumes 8 cinder -# Uncomment below to enable mgr modules -# For a list of available modules: -# http://docs.ceph.com/docs/master/mgr/ -# This overrides mgr_initial_modules (default: restful, status) -# Any module not listed here will be disabled -ceph_mgr_enabled_modules: - - restful - - status - - prometheus - - balancer - - iostat - - pg_autoscaler - -# You can configure your mgr modules -# below. Each module has its own set -# of key/value. Refer to the doc -# above for more info. For example: -ceph_mgr_modules_config: -# balancer: -# active: 1 -# prometheus: - # server_port: 9283 -# server_addr: 0.0.0.0 -# dashboard: -# port: 7000 -# localpool: -# failure_domain: host -# subtree: rack -# pg_num: "128" -# num_rep: "3" -# min_size: "2" - endpoints: cluster_domain_suffix: cluster.local local_image_registry: @@ -614,26 +541,17 @@ endpoints: scheme: default: http -monitoring: - prometheus: - enabled: true - ceph_mgr: - scrape: true - port: 9283 - manifests: configmap_bin: true configmap_test_bin: true configmap_etc: true deployment_mds: true - deployment_mgr: true deployment_checkdns: true job_bootstrap: false job_ns_client_ceph_config: true job_cephfs_client_key: true job_image_repo_sync: true job_rbd_pool: true - service_mgr: true helm_tests: true cronjob_checkPGs: true cronjob_defragosds: true diff --git a/ceph-client/values_overrides/apparmor.yaml b/ceph-client/values_overrides/apparmor.yaml index e643dfd60..21adebd6c 100644 --- a/ceph-client/values_overrides/apparmor.yaml +++ b/ceph-client/values_overrides/apparmor.yaml @@ -8,9 +8,6 @@ pod: ceph-mds: ceph-mds: runtime/default ceph-init-dirs: runtime/default - ceph-mgr: - ceph-mgr: runtime/default - ceph-init-dirs: runtime/default ceph-rbd-pool: ceph-rbd-pool: runtime/default init: runtime/default diff --git a/ceph-mon/Chart.yaml b/ceph-mon/Chart.yaml index e8e3c4ca2..85fbc9d7c 100644 --- a/ceph-mon/Chart.yaml +++ b/ceph-mon/Chart.yaml @@ -15,6 +15,6 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Ceph Mon name: ceph-mon -version: 0.1.17 +version: 0.1.18 home: https://github.com/ceph/ceph ... diff --git a/ceph-client/templates/bin/mgr/_check.sh.tpl b/ceph-mon/templates/bin/mgr/_check.sh.tpl similarity index 100% rename from ceph-client/templates/bin/mgr/_check.sh.tpl rename to ceph-mon/templates/bin/mgr/_check.sh.tpl diff --git a/ceph-client/templates/bin/mgr/_start.sh.tpl b/ceph-mon/templates/bin/mgr/_start.sh.tpl similarity index 97% rename from ceph-client/templates/bin/mgr/_start.sh.tpl rename to ceph-mon/templates/bin/mgr/_start.sh.tpl index 64e273b0c..4de8de112 100644 --- a/ceph-client/templates/bin/mgr/_start.sh.tpl +++ b/ceph-mon/templates/bin/mgr/_start.sh.tpl @@ -6,7 +6,7 @@ set -ex : "${ADMIN_KEYRING:=/etc/ceph/${CLUSTER}.client.admin.keyring}" : "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}" -{{ include "ceph-client.snippets.mon_host_from_k8s_ep" . }} +{{ include "ceph-mon.snippets.mon_host_from_k8s_ep" . }} if [[ ! -e ${CEPH_CONF}.template ]]; then echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon" diff --git a/ceph-client/templates/bin/utils/_checkPGs.py.tpl b/ceph-mon/templates/bin/utils/_checkPGs.py.tpl similarity index 100% rename from ceph-client/templates/bin/utils/_checkPGs.py.tpl rename to ceph-mon/templates/bin/utils/_checkPGs.py.tpl diff --git a/ceph-mon/templates/configmap-bin.yaml b/ceph-mon/templates/configmap-bin.yaml index d433cd335..438b1fe64 100644 --- a/ceph-mon/templates/configmap-bin.yaml +++ b/ceph-mon/templates/configmap-bin.yaml @@ -47,6 +47,11 @@ data: mon-check.sh: | {{ tuple "bin/mon/_check.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + mgr-start.sh: | +{{ tuple "bin/mgr/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + mgr-check.sh: | +{{ tuple "bin/mgr/_check.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + moncheck-start.sh: | {{ tuple "bin/moncheck/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} moncheck-reap-zombies.py: | @@ -57,3 +62,6 @@ data: utils-checkDNS.sh: | {{ tuple "bin/utils/_checkDNS.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- end }} + + utils-checkPGs.py: | +{{ tuple "bin/utils/_checkPGs.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/ceph-client/templates/deployment-mgr.yaml b/ceph-mon/templates/deployment-mgr.yaml similarity index 89% rename from ceph-client/templates/deployment-mgr.yaml rename to ceph-mon/templates/deployment-mgr.yaml index e53fe29e4..63743fb06 100644 --- a/ceph-client/templates/deployment-mgr.yaml +++ b/ceph-mon/templates/deployment-mgr.yaml @@ -40,7 +40,7 @@ spec: annotations: {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} - configmap-etc-client-hash: {{ tuple "configmap-etc-client.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} {{ dict "envAll" $envAll "podName" "ceph-mgr" "containerNames" (list "ceph-mgr" "ceph-init-dirs") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} spec: {{ dict "envAll" $envAll "application" "mgr" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} @@ -56,7 +56,7 @@ spec: initContainers: {{ tuple $envAll "mgr" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} - name: ceph-init-dirs -{{ tuple $envAll "ceph_mds" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll "ceph_mgr" | include "helm-toolkit.snippets.image" | indent 10 }} {{ dict "envAll" $envAll "application" "mgr" "container" "init_dirs" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} command: - /tmp/init-dirs.sh @@ -70,7 +70,7 @@ spec: mountPath: /run - name: pod-etc-ceph mountPath: /etc/ceph - - name: ceph-client-bin + - name: ceph-mon-bin mountPath: /tmp/init-dirs.sh subPath: init-dirs.sh readOnly: true @@ -142,23 +142,19 @@ spec: mountPath: /run - name: pod-etc-ceph mountPath: /etc/ceph - - name: ceph-client-bin + - name: ceph-mon-bin mountPath: /mgr-start.sh subPath: mgr-start.sh readOnly: true - - name: ceph-client-bin + - name: ceph-mon-bin mountPath: /tmp/mgr-check.sh subPath: mgr-check.sh readOnly: true - - name: ceph-client-bin - mountPath: /tmp/utils-checkDNS.sh - subPath: utils-checkDNS.sh - readOnly: true - - name: ceph-client-etc + - name: ceph-mon-etc mountPath: /etc/ceph/ceph.conf.template subPath: ceph.conf readOnly: true - - name: ceph-client-admin-keyring + - name: ceph-mon-admin-keyring mountPath: /etc/ceph/ceph.client.admin.keyring subPath: ceph.client.admin.keyring readOnly: true @@ -172,14 +168,10 @@ spec: - name: pod-var-lib-ceph-crash mountPath: /var/lib/ceph/crash readOnly: false - - name: ceph-client-bin + - name: ceph-mon-bin mountPath: /tmp/utils-checkPGs.py subPath: utils-checkPGs.py readOnly: true - - name: ceph-client-bin - mountPath: /tmp/utils-checkPGs.sh - subPath: utils-checkPGs.sh - readOnly: true volumes: - name: pod-tmp emptyDir: {} @@ -188,13 +180,13 @@ spec: medium: "Memory" - name: pod-etc-ceph emptyDir: {} - - name: ceph-client-bin + - name: ceph-mon-bin configMap: - name: ceph-client-bin + name: ceph-mon-bin defaultMode: 0555 - - name: ceph-client-etc + - name: ceph-mon-etc configMap: - name: ceph-client-etc + name: ceph-mon-etc defaultMode: 0444 - name: pod-var-lib-ceph emptyDir: {} @@ -202,7 +194,7 @@ spec: hostPath: path: /var/lib/openstack-helm/ceph/crash type: DirectoryOrCreate - - name: ceph-client-admin-keyring + - name: ceph-mon-admin-keyring secret: secretName: {{ .Values.secrets.keyrings.admin }} - name: ceph-bootstrap-mgr-keyring diff --git a/ceph-client/templates/service-mgr.yaml b/ceph-mon/templates/service-mgr.yaml similarity index 100% rename from ceph-client/templates/service-mgr.yaml rename to ceph-mon/templates/service-mgr.yaml diff --git a/ceph-mon/templates/snippets/_mon_host_from_k8s_ep.sh.tpl b/ceph-mon/templates/snippets/_mon_host_from_k8s_ep.sh.tpl new file mode 100644 index 000000000..eb7189825 --- /dev/null +++ b/ceph-mon/templates/snippets/_mon_host_from_k8s_ep.sh.tpl @@ -0,0 +1,68 @@ +{{- define "ceph-mon.snippets.mon_host_from_k8s_ep" -}} +{{/* + +Inserts a bash function definition mon_host_from_k8s_ep() which can be used +to construct a mon_hosts value from the given namespaced endpoint. + +Usage (e.g. in _script.sh.tpl): + #!/bin/bash + + : "${NS:=ceph}" + : "${EP:=ceph-mon-discovery}" + + {{ include "ceph-mon.snippets.mon_host_from_k8s_ep" . }} + + MON_HOST=$(mon_host_from_k8s_ep "$NS" "$EP") + + if [ -z "$MON_HOST" ]; then + # deal with failure + else + sed -i -e "s/^mon_host = /mon_host = $MON_HOST/" /etc/ceph/ceph.conf + fi +*/}} +{{` +# Construct a mon_hosts value from the given namespaced endpoint +# IP x.x.x.x with port p named "mon-msgr2" will appear as [v2:x.x.x.x/p/0] +# IP x.x.x.x with port q named "mon" will appear as [v1:x.x.x.x/q/0] +# IP x.x.x.x with ports p and q will appear as [v2:x.x.x.x/p/0,v1:x.x.x.x/q/0] +# The entries for all IPs will be joined with commas +mon_host_from_k8s_ep() { + local ns=$1 + local ep=$2 + + if [ -z "$ns" ] || [ -z "$ep" ]; then + return 1 + fi + + # We don't want shell expansion for the go-template expression + # shellcheck disable=SC2016 + kubectl get endpoints -n "$ns" "$ep" -o go-template=' + {{- $sep := "" }} + {{- range $_,$s := .subsets }} + {{- $v2port := 0 }} + {{- $v1port := 0 }} + {{- range $_,$port := index $s "ports" }} + {{- if (eq $port.name "mon-msgr2") }} + {{- $v2port = $port.port }} + {{- else if (eq $port.name "mon") }} + {{- $v1port = $port.port }} + {{- end }} + {{- end }} + {{- range $_,$address := index $s "addresses" }} + {{- $v2endpoint := printf "v2:%s:%d/0" $address.ip $v2port }} + {{- $v1endpoint := printf "v1:%s:%d/0" $address.ip $v1port }} + {{- if (and $v2port $v1port) }} + {{- printf "%s[%s,%s]" $sep $v2endpoint $v1endpoint }} + {{- $sep = "," }} + {{- else if $v2port }} + {{- printf "%s[%s]" $sep $v2endpoint }} + {{- $sep = "," }} + {{- else if $v1port }} + {{- printf "%s[%s]" $sep $v1endpoint }} + {{- $sep = "," }} + {{- end }} + {{- end }} + {{- end }}' +} +`}} +{{- end -}} diff --git a/ceph-mon/values.yaml b/ceph-mon/values.yaml index f060c13a6..172069377 100644 --- a/ceph-mon/values.yaml +++ b/ceph-mon/values.yaml @@ -26,6 +26,7 @@ images: ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:change_770201_ubuntu_bionic-20210113' ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:change_770201_ubuntu_bionic-20210113' ceph_mon: 'docker.io/openstackhelm/ceph-daemon:change_770201_ubuntu_bionic-20210113' + ceph_mgr: 'docker.io/openstackhelm/ceph-daemon:change_770201_ubuntu_bionic-20210113' ceph_mon_check: 'docker.io/openstackhelm/ceph-config-helper:change_770201_ubuntu_bionic-20210113' dep_check: 'quay.io/airshipit/kubernetes-entrypoint:v1.0.0' image_repo_sync: 'docker.io/library/docker:17.07.0' @@ -42,6 +43,9 @@ labels: mon: node_selector_key: ceph-mon node_selector_value: enabled + mgr: + node_selector_key: ceph-mgr + node_selector_value: enabled pod: security_context: @@ -59,6 +63,17 @@ pod: runAsUser: 64045 readOnlyRootFilesystem: true allowPrivilegeEscalation: false + mgr: + pod: + runAsUser: 65534 + container: + init_dirs: + runAsUser: 0 + readOnlyRootFilesystem: true + mgr: + runAsUser: 64045 + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false moncheck: pod: runAsUser: 65534 @@ -98,6 +113,7 @@ pod: readOnlyRootFilesystem: true dns_policy: "ClusterFirstWithHostNet" replicas: + mgr: 2 mon_check: 1 lifecycle: upgrades: @@ -107,6 +123,9 @@ pod: enabled: true min_ready_seconds: 0 max_unavailable: 1 + updateStrategy: + mgr: + type: Recreate affinity: anti: type: @@ -124,6 +143,13 @@ pod: limits: memory: "100Mi" cpu: "500m" + mgr: + requests: + memory: "5Mi" + cpu: "250m" + limits: + memory: "50Mi" + cpu: "500m" mon_check: requests: memory: "5Mi" @@ -154,6 +180,16 @@ pod: memory: "1024Mi" cpu: "2000m" tolerations: + mgr: + tolerations: + - effect: NoExecute + key: node.kubernetes.io/not-ready + operator: Exists + tolerationSeconds: 60 + - effect: NoExecute + key: node.kubernetes.io/unreachable + operator: Exists + tolerationSeconds: 60 mon_check: tolerations: - effect: NoExecute @@ -178,6 +214,8 @@ network: cluster: 192.168.0.0/16 conf: + features: + mgr: true templates: keyring: admin: | @@ -272,6 +310,13 @@ dependencies: jobs: - ceph-storage-keys-generator - ceph-mon-keyring-generator + mgr: + jobs: + - ceph-storage-keys-generator + - ceph-mgr-keyring-generator + services: + - endpoint: internal + service: ceph_mon moncheck: jobs: - ceph-storage-keys-generator @@ -298,6 +343,38 @@ bootstrap: } #ensure_pool volumes 8 cinder +# Uncomment below to enable mgr modules +# For a list of available modules: +# http://docs.ceph.com/docs/master/mgr/ +# This overrides mgr_initial_modules (default: restful, status) +# Any module not listed here will be disabled +ceph_mgr_enabled_modules: + - restful + - status + - prometheus + - balancer + - iostat + - pg_autoscaler + +# You can configure your mgr modules +# below. Each module has its own set +# of key/value. Refer to the doc +# above for more info. For example: +ceph_mgr_modules_config: +# balancer: +# active: 1 +# prometheus: + # server_port: 9283 +# server_addr: 0.0.0.0 +# dashboard: +# port: 7000 +# localpool: +# failure_domain: host +# subtree: rack +# pg_num: "128" +# num_rep: "3" +# min_size: "2" + # if you change provision_storage_class to false # it is presumed you manage your own storage # class definition externally @@ -344,17 +421,39 @@ endpoints: default: 6789 mon_msgr2: default: 3300 + ceph_mgr: + namespace: null + hosts: + default: ceph-mgr + host_fqdn_override: + default: null + port: + mgr: + default: 7000 + metrics: + default: 9283 + scheme: + default: http + +monitoring: + prometheus: + enabled: true + ceph_mgr: + scrape: true + port: 9283 manifests: configmap_bin: true configmap_etc: true configmap_templates: true daemonset_mon: true + deployment_mgr: true deployment_moncheck: true job_image_repo_sync: true job_bootstrap: true job_keyring: true service_mon: true + service_mgr: true service_mon_discovery: true job_storage_admin_keys: true ... diff --git a/ceph-mon/values_overrides/apparmor.yaml b/ceph-mon/values_overrides/apparmor.yaml index 250703bce..e6aeea56e 100644 --- a/ceph-mon/values_overrides/apparmor.yaml +++ b/ceph-mon/values_overrides/apparmor.yaml @@ -6,6 +6,9 @@ pod: ceph-init-dirs: runtime/default ceph-mon: runtime/default ceph-log-ownership: runtime/default + ceph-mgr: + ceph-mgr: runtime/default + ceph-init-dirs: runtime/default ceph-mon-check: ceph-mon: runtime/default init: runtime/default diff --git a/releasenotes/notes/ceph-client.yaml b/releasenotes/notes/ceph-client.yaml index 6e013706d..3cfa9e309 100644 --- a/releasenotes/notes/ceph-client.yaml +++ b/releasenotes/notes/ceph-client.yaml @@ -30,4 +30,5 @@ ceph-client: - 0.1.27 Update ceph_mon config to ips from fqdn - 0.1.28 Fix ceph.conf update job labels, rendering - 0.1.29 Consolidate mon_host discovery + - 0.1.30 Move ceph-mgr deployment to the ceph-mon chart ... diff --git a/releasenotes/notes/ceph-mon.yaml b/releasenotes/notes/ceph-mon.yaml index 5e491f7a0..389eb9c40 100644 --- a/releasenotes/notes/ceph-mon.yaml +++ b/releasenotes/notes/ceph-mon.yaml @@ -18,4 +18,5 @@ ceph-mon: - 0.1.15 Prevent mon-check from removing mons when down temporarily - 0.1.16 Correct Ceph Mon Check Ports - 0.1.17 Skip monmap endpoint check for missing mons + - 0.1.18 Move ceph-mgr deployment to the ceph-mon chart ...