Move ceph-mgr deployment to the ceph-mon chart

This change moves the ceph-mgr deployment from the ceph-client
chart to the ceph-mon chart. Its purpose is to facilitate the
proper Ceph upgrade procedure, which prescribes restarting mgr
daemons before mon daemons.

There will be additional work required to implement the correct
daemon restart procedure for upgrades. This change only addresses
the move of the ceph-mgr deployment.

Change-Id: I3ac4a75f776760425c88a0ba1edae5fb339f128d

ceph-client/Chart.yaml

@@ -15,6 +15,6 @@ apiVersion: v1
 appVersion: v1.0.0
 description: OpenStack-Helm Ceph Client
 name: ceph-client
-version: 0.1.29
+version: 0.1.30
 home: https://github.com/ceph/ceph-client
 ...

ceph-client/templates/configmap-bin.yaml

@@ -43,11 +43,6 @@ data:
   mds-start.sh: |
 {{ tuple "bin/mds/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 
-  mgr-start.sh: |
-{{ tuple "bin/mgr/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-  mgr-check.sh: |
-{{ tuple "bin/mgr/_check.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-
   helm-tests.sh: |
 {{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   utils-checkDNS.sh: |
@@ -55,8 +50,6 @@ data:
   utils-checkDNS_start.sh: |
 {{ tuple "bin/utils/_checkDNS_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 
-  utils-checkPGs.py: |
-{{ tuple "bin/utils/_checkPGs.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   utils-checkPGs.sh: |
 {{ tuple "bin/utils/_checkPGs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 

ceph-client/templates/cronjob-checkPGs.yaml

@@ -91,10 +91,6 @@ spec:
                 mountPath: /tmp
               - name: pod-etc-ceph
                 mountPath: /etc/ceph
-              - name: ceph-client-bin
-                mountPath: /tmp/utils-checkPGs.py
-                subPath: utils-checkPGs.py
-                readOnly: true
               - name: ceph-client-bin
                 mountPath: /tmp/utils-checkPGs.sh
                 subPath: utils-checkPGs.sh

ceph-client/values.yaml

@@ -27,7 +27,6 @@ images:
     ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:change_770201_ubuntu_bionic-20210113'
     ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:change_770201_ubuntu_bionic-20210113'
     ceph_mds: 'docker.io/openstackhelm/ceph-daemon:change_770201_ubuntu_bionic-20210113'
-    ceph_mgr: 'docker.io/openstackhelm/ceph-daemon:change_770201_ubuntu_bionic-20210113'
     ceph_rbd_pool: 'docker.io/openstackhelm/ceph-config-helper:change_770201_ubuntu_bionic-20210113'
     dep_check: 'quay.io/airshipit/kubernetes-entrypoint:v1.0.0'
     image_repo_sync: 'docker.io/library/docker:17.07.0'
@@ -44,12 +43,12 @@ labels:
   test:
     node_selector_key: openstack-control-plane
     node_selector_value: enabled
-  mds:
-    node_selector_key: ceph-mds
-    node_selector_value: enabled
   mgr:
     node_selector_key: ceph-mgr
     node_selector_value: enabled
+  mds:
+    node_selector_key: ceph-mds
+    node_selector_value: enabled
   checkdns:
     node_selector_key: ceph-mon
     node_selector_value: enabled
@@ -74,17 +73,6 @@ pod:
           runAsUser: 64045
           readOnlyRootFilesystem: true
           allowPrivilegeEscalation: false
-    mgr:
-      pod:
-        runAsUser: 65534
-      container:
-        init_dirs:
-          runAsUser: 0
-          readOnlyRootFilesystem: true
-        mgr:
-          runAsUser: 64045
-          readOnlyRootFilesystem: true
-          allowPrivilegeEscalation: false
     bootstrap:
       pod:
         runAsUser: 65534
@@ -109,7 +97,6 @@ pod:
   dns_policy: "ClusterFirstWithHostNet"
   replicas:
     mds: 2
-    mgr: 2
   lifecycle:
     upgrades:
       deployments:
@@ -118,9 +105,6 @@ pod:
         rolling_update:
           max_surge: 25%
           max_unavailable: 25%
-  updateStrategy:
-    mgr:
-      type: Recreate
   affinity:
     anti:
       type:
@@ -138,13 +122,6 @@ pod:
       limits:
         memory: "50Mi"
         cpu: "500m"
-    mgr:
-      requests:
-        memory: "5Mi"
-        cpu: "250m"
-      limits:
-        memory: "50Mi"
-        cpu: "500m"
     checkdns:
       requests:
         memory: "5Mi"
@@ -202,16 +179,6 @@ pod:
         key: node.kubernetes.io/unreachable
         operator: Exists
         tolerationSeconds: 60
-    mgr:
-      tolerations:
-      - effect: NoExecute
-        key: node.kubernetes.io/not-ready
-        operator: Exists
-        tolerationSeconds: 60
-      - effect: NoExecute
-        key: node.kubernetes.io/unreachable
-        operator: Exists
-        tolerationSeconds: 60
 
 secrets:
   keyrings:
@@ -259,7 +226,6 @@ jobs:
 conf:
   features:
     mds: true
-    mgr: true
     pg_autoscaler: true
     cluster_flags:
       # List of flags to set or unset separated by spaces
@@ -489,13 +455,6 @@ dependencies:
       services:
         - endpoint: internal
           service: ceph_mon
-    mgr:
-      jobs:
-        - ceph-storage-keys-generator
-        - ceph-mgr-keyring-generator
-      services:
-        - endpoint: internal
-          service: ceph_mon
     pool_checkpgs:
       jobs:
         - ceph-rbd-pool
@@ -542,38 +501,6 @@ bootstrap:
     }
     #ensure_pool volumes 8 cinder
 
-# Uncomment below to enable mgr modules
-# For a list of available modules:
-#  http://docs.ceph.com/docs/master/mgr/
-# This overrides mgr_initial_modules (default: restful, status)
-# Any module not listed here will be disabled
-ceph_mgr_enabled_modules:
-  - restful
-  - status
-  - prometheus
-  - balancer
-  - iostat
-  - pg_autoscaler
-
-# You can configure your mgr modules
-# below. Each module has its own set
-# of key/value. Refer to the doc
-# above for more info. For example:
-ceph_mgr_modules_config:
-#  balancer:
-#    active: 1
-#  prometheus:
-    # server_port: 9283
-#    server_addr: 0.0.0.0
-#  dashboard:
-#    port: 7000
-#  localpool:
-#    failure_domain: host
-#    subtree: rack
-#    pg_num: "128"
-#    num_rep: "3"
-#    min_size: "2"
-
 endpoints:
   cluster_domain_suffix: cluster.local
   local_image_registry:
@@ -614,26 +541,17 @@ endpoints:
     scheme:
       default: http
 
-monitoring:
-  prometheus:
-    enabled: true
-    ceph_mgr:
-      scrape: true
-      port: 9283
-
 manifests:
   configmap_bin: true
   configmap_test_bin: true
   configmap_etc: true
   deployment_mds: true
-  deployment_mgr: true
   deployment_checkdns: true
   job_bootstrap: false
   job_ns_client_ceph_config: true
   job_cephfs_client_key: true
   job_image_repo_sync: true
   job_rbd_pool: true
-  service_mgr: true
   helm_tests: true
   cronjob_checkPGs: true
   cronjob_defragosds: true

ceph-client/values_overrides/apparmor.yaml

@@ -8,9 +8,6 @@ pod:
     ceph-mds:
       ceph-mds: runtime/default
       ceph-init-dirs: runtime/default
-    ceph-mgr:
-      ceph-mgr: runtime/default
-      ceph-init-dirs: runtime/default
     ceph-rbd-pool:
       ceph-rbd-pool: runtime/default
       init: runtime/default

ceph-mon/Chart.yaml

@@ -15,6 +15,6 @@ apiVersion: v1
 appVersion: v1.0.0
 description: OpenStack-Helm Ceph Mon
 name: ceph-mon
-version: 0.1.17
+version: 0.1.18
 home: https://github.com/ceph/ceph
 ...

ceph-mon/templates/bin/mgr/_start.sh.tpl

@@ -6,7 +6,7 @@ set -ex
 : "${ADMIN_KEYRING:=/etc/ceph/${CLUSTER}.client.admin.keyring}"
 : "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}"
 
-{{ include "ceph-client.snippets.mon_host_from_k8s_ep" . }}
+{{ include "ceph-mon.snippets.mon_host_from_k8s_ep" . }}
 
 if [[ ! -e ${CEPH_CONF}.template ]]; then
   echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon"

ceph-mon/templates/configmap-bin.yaml

@@ -47,6 +47,11 @@ data:
   mon-check.sh: |
 {{ tuple "bin/mon/_check.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 
+  mgr-start.sh: |
+{{ tuple "bin/mgr/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  mgr-check.sh: |
+{{ tuple "bin/mgr/_check.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+
   moncheck-start.sh: |
 {{ tuple "bin/moncheck/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   moncheck-reap-zombies.py: |
@@ -57,3 +62,6 @@ data:
   utils-checkDNS.sh: |
 {{ tuple "bin/utils/_checkDNS.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- end }}
+
+  utils-checkPGs.py: |
+{{ tuple "bin/utils/_checkPGs.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

ceph-mon/templates/deployment-mgr.yaml

@@ -40,7 +40,7 @@ spec:
       annotations:
 {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
         configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
-        configmap-etc-client-hash: {{ tuple "configmap-etc-client.yaml" . | include "helm-toolkit.utils.hash" }}
+        configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
 {{ dict "envAll" $envAll "podName" "ceph-mgr" "containerNames" (list "ceph-mgr" "ceph-init-dirs") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
     spec:
 {{ dict "envAll" $envAll "application" "mgr" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
@@ -56,7 +56,7 @@ spec:
       initContainers:
 {{ tuple $envAll "mgr" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
         - name: ceph-init-dirs
-{{ tuple $envAll "ceph_mds" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll "ceph_mgr" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ dict "envAll" $envAll "application" "mgr" "container" "init_dirs" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
           command:
             - /tmp/init-dirs.sh
@@ -70,7 +70,7 @@ spec:
               mountPath: /run
             - name: pod-etc-ceph
               mountPath: /etc/ceph
-            - name: ceph-client-bin
+            - name: ceph-mon-bin
               mountPath: /tmp/init-dirs.sh
               subPath: init-dirs.sh
               readOnly: true
@@ -142,23 +142,19 @@ spec:
               mountPath: /run
             - name: pod-etc-ceph
               mountPath: /etc/ceph
-            - name: ceph-client-bin
+            - name: ceph-mon-bin
               mountPath: /mgr-start.sh
               subPath: mgr-start.sh
               readOnly: true
-            - name: ceph-client-bin
+            - name: ceph-mon-bin
               mountPath: /tmp/mgr-check.sh
               subPath: mgr-check.sh
               readOnly: true
-            - name: ceph-client-bin
+            - name: ceph-mon-etc
-              mountPath: /tmp/utils-checkDNS.sh
-              subPath: utils-checkDNS.sh
-              readOnly: true
-            - name: ceph-client-etc
               mountPath: /etc/ceph/ceph.conf.template
               subPath: ceph.conf
               readOnly: true
-            - name: ceph-client-admin-keyring
+            - name: ceph-mon-admin-keyring
               mountPath: /etc/ceph/ceph.client.admin.keyring
               subPath: ceph.client.admin.keyring
               readOnly: true
@@ -172,14 +168,10 @@ spec:
             - name: pod-var-lib-ceph-crash
               mountPath: /var/lib/ceph/crash
               readOnly: false
-            - name: ceph-client-bin
+            - name: ceph-mon-bin
               mountPath: /tmp/utils-checkPGs.py
               subPath: utils-checkPGs.py
               readOnly: true
-            - name: ceph-client-bin
-              mountPath: /tmp/utils-checkPGs.sh
-              subPath: utils-checkPGs.sh
-              readOnly: true
       volumes:
         - name: pod-tmp
           emptyDir: {}
@@ -188,13 +180,13 @@ spec:
             medium: "Memory"
         - name: pod-etc-ceph
           emptyDir: {}
-        - name: ceph-client-bin
+        - name: ceph-mon-bin
           configMap:
-            name: ceph-client-bin
+            name: ceph-mon-bin
             defaultMode: 0555
-        - name: ceph-client-etc
+        - name: ceph-mon-etc
           configMap:
-            name: ceph-client-etc
+            name: ceph-mon-etc
             defaultMode: 0444
         - name: pod-var-lib-ceph
           emptyDir: {}
@@ -202,7 +194,7 @@ spec:
           hostPath:
             path: /var/lib/openstack-helm/ceph/crash
             type: DirectoryOrCreate
-        - name: ceph-client-admin-keyring
+        - name: ceph-mon-admin-keyring
           secret:
             secretName: {{ .Values.secrets.keyrings.admin }}
         - name: ceph-bootstrap-mgr-keyring

ceph-mon/templates/snippets/_mon_host_from_k8s_ep.sh.tpl

@@ -0,0 +1,68 @@
+{{- define "ceph-mon.snippets.mon_host_from_k8s_ep" -}}
+{{/*
+
+Inserts a bash function definition mon_host_from_k8s_ep() which can be used
+to construct a mon_hosts value from the given namespaced endpoint.
+
+Usage (e.g. in _script.sh.tpl):
+    #!/bin/bash
+
+    : "${NS:=ceph}"
+    : "${EP:=ceph-mon-discovery}"
+
+    {{ include "ceph-mon.snippets.mon_host_from_k8s_ep" . }}
+
+    MON_HOST=$(mon_host_from_k8s_ep "$NS" "$EP")
+
+    if [ -z "$MON_HOST" ]; then
+        # deal with failure
+    else
+        sed -i -e "s/^mon_host = /mon_host = $MON_HOST/" /etc/ceph/ceph.conf
+    fi
+*/}}
+{{`
+# Construct a mon_hosts value from the given namespaced endpoint
+# IP x.x.x.x with port p named "mon-msgr2" will appear as [v2:x.x.x.x/p/0]
+# IP x.x.x.x with port q named "mon" will appear as [v1:x.x.x.x/q/0]
+# IP x.x.x.x with ports p and q will appear as [v2:x.x.x.x/p/0,v1:x.x.x.x/q/0]
+# The entries for all IPs will be joined with commas
+mon_host_from_k8s_ep() {
+  local ns=$1
+  local ep=$2
+
+  if [ -z "$ns" ] || [ -z "$ep" ]; then
+    return 1
+  fi
+
+  # We don't want shell expansion for the go-template expression
+  # shellcheck disable=SC2016
+  kubectl get endpoints -n "$ns" "$ep" -o go-template='
+    {{- $sep := "" }}
+    {{- range $_,$s := .subsets }}
+      {{- $v2port := 0 }}
+      {{- $v1port := 0 }}
+      {{- range $_,$port := index $s "ports" }}
+        {{- if (eq $port.name "mon-msgr2") }}
+          {{- $v2port = $port.port }}
+        {{- else if (eq $port.name "mon") }}
+          {{- $v1port = $port.port }}
+        {{- end }}
+      {{- end }}
+      {{- range $_,$address := index $s "addresses" }}
+        {{- $v2endpoint := printf "v2:%s:%d/0" $address.ip $v2port }}
+        {{- $v1endpoint := printf "v1:%s:%d/0" $address.ip $v1port }}
+        {{- if (and $v2port $v1port) }}
+          {{- printf "%s[%s,%s]" $sep $v2endpoint $v1endpoint }}
+          {{- $sep = "," }}
+        {{- else if $v2port }}
+          {{- printf "%s[%s]" $sep $v2endpoint }}
+          {{- $sep = "," }}
+        {{- else if $v1port }}
+          {{- printf "%s[%s]" $sep $v1endpoint }}
+          {{- $sep = "," }}
+        {{- end }}
+      {{- end }}
+    {{- end }}'
+}
+`}}
+{{- end -}}

ceph-mon/values.yaml

@@ -26,6 +26,7 @@ images:
     ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:change_770201_ubuntu_bionic-20210113'
     ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:change_770201_ubuntu_bionic-20210113'
     ceph_mon: 'docker.io/openstackhelm/ceph-daemon:change_770201_ubuntu_bionic-20210113'
+    ceph_mgr: 'docker.io/openstackhelm/ceph-daemon:change_770201_ubuntu_bionic-20210113'
     ceph_mon_check: 'docker.io/openstackhelm/ceph-config-helper:change_770201_ubuntu_bionic-20210113'
     dep_check: 'quay.io/airshipit/kubernetes-entrypoint:v1.0.0'
     image_repo_sync: 'docker.io/library/docker:17.07.0'
@@ -42,6 +43,9 @@ labels:
   mon:
     node_selector_key: ceph-mon
     node_selector_value: enabled
+  mgr:
+    node_selector_key: ceph-mgr
+    node_selector_value: enabled
 
 pod:
   security_context:
@@ -59,6 +63,17 @@ pod:
           runAsUser: 64045
           readOnlyRootFilesystem: true
           allowPrivilegeEscalation: false
+    mgr:
+      pod:
+        runAsUser: 65534
+      container:
+        init_dirs:
+          runAsUser: 0
+          readOnlyRootFilesystem: true
+        mgr:
+          runAsUser: 64045
+          readOnlyRootFilesystem: true
+          allowPrivilegeEscalation: false
     moncheck:
       pod:
         runAsUser: 65534
@@ -98,6 +113,7 @@ pod:
           readOnlyRootFilesystem: true
   dns_policy: "ClusterFirstWithHostNet"
   replicas:
+    mgr: 2
     mon_check: 1
   lifecycle:
     upgrades:
@@ -107,6 +123,9 @@ pod:
           enabled: true
           min_ready_seconds: 0
           max_unavailable: 1
+  updateStrategy:
+    mgr:
+      type: Recreate
   affinity:
     anti:
       type:
@@ -124,6 +143,13 @@ pod:
       limits:
         memory: "100Mi"
         cpu: "500m"
+    mgr:
+      requests:
+        memory: "5Mi"
+        cpu: "250m"
+      limits:
+        memory: "50Mi"
+        cpu: "500m"
     mon_check:
       requests:
         memory: "5Mi"
@@ -154,6 +180,16 @@ pod:
           memory: "1024Mi"
           cpu: "2000m"
   tolerations:
+    mgr:
+      tolerations:
+      - effect: NoExecute
+        key: node.kubernetes.io/not-ready
+        operator: Exists
+        tolerationSeconds: 60
+      - effect: NoExecute
+        key: node.kubernetes.io/unreachable
+        operator: Exists
+        tolerationSeconds: 60
     mon_check:
       tolerations:
       - effect: NoExecute
@@ -178,6 +214,8 @@ network:
   cluster: 192.168.0.0/16
 
 conf:
+  features:
+    mgr: true
   templates:
     keyring:
       admin: |
@@ -272,6 +310,13 @@ dependencies:
       jobs:
         - ceph-storage-keys-generator
         - ceph-mon-keyring-generator
+    mgr:
+      jobs:
+        - ceph-storage-keys-generator
+        - ceph-mgr-keyring-generator
+      services:
+        - endpoint: internal
+          service: ceph_mon
     moncheck:
       jobs:
         - ceph-storage-keys-generator
@@ -298,6 +343,38 @@ bootstrap:
     }
     #ensure_pool volumes 8 cinder
 
+# Uncomment below to enable mgr modules
+# For a list of available modules:
+#  http://docs.ceph.com/docs/master/mgr/
+# This overrides mgr_initial_modules (default: restful, status)
+# Any module not listed here will be disabled
+ceph_mgr_enabled_modules:
+  - restful
+  - status
+  - prometheus
+  - balancer
+  - iostat
+  - pg_autoscaler
+
+# You can configure your mgr modules
+# below. Each module has its own set
+# of key/value. Refer to the doc
+# above for more info. For example:
+ceph_mgr_modules_config:
+#  balancer:
+#    active: 1
+#  prometheus:
+    # server_port: 9283
+#    server_addr: 0.0.0.0
+#  dashboard:
+#    port: 7000
+#  localpool:
+#    failure_domain: host
+#    subtree: rack
+#    pg_num: "128"
+#    num_rep: "3"
+#    min_size: "2"
+
 # if you change provision_storage_class to false
 # it is presumed you manage your own storage
 # class definition externally
@@ -344,17 +421,39 @@ endpoints:
         default: 6789
       mon_msgr2:
         default: 3300
+  ceph_mgr:
+    namespace: null
+    hosts:
+      default: ceph-mgr
+    host_fqdn_override:
+      default: null
+    port:
+      mgr:
+        default: 7000
+      metrics:
+        default: 9283
+    scheme:
+      default: http
+
+monitoring:
+  prometheus:
+    enabled: true
+    ceph_mgr:
+      scrape: true
+      port: 9283
 
 manifests:
   configmap_bin: true
   configmap_etc: true
   configmap_templates: true
   daemonset_mon: true
+  deployment_mgr: true
   deployment_moncheck: true
   job_image_repo_sync: true
   job_bootstrap: true
   job_keyring: true
   service_mon: true
+  service_mgr: true
   service_mon_discovery: true
   job_storage_admin_keys: true
 ...

ceph-mon/values_overrides/apparmor.yaml

@@ -6,6 +6,9 @@ pod:
       ceph-init-dirs: runtime/default
       ceph-mon: runtime/default
       ceph-log-ownership: runtime/default
+    ceph-mgr:
+      ceph-mgr: runtime/default
+      ceph-init-dirs: runtime/default
     ceph-mon-check:
       ceph-mon: runtime/default
       init: runtime/default

releasenotes/notes/ceph-client.yaml

@@ -30,4 +30,5 @@ ceph-client:
   - 0.1.27 Update ceph_mon config to ips from fqdn
   - 0.1.28 Fix ceph.conf update job labels, rendering
   - 0.1.29 Consolidate mon_host discovery
+  - 0.1.30 Move ceph-mgr deployment to the ceph-mon chart
 ...

releasenotes/notes/ceph-mon.yaml

@@ -18,4 +18,5 @@ ceph-mon:
   - 0.1.15 Prevent mon-check from removing mons when down temporarily
   - 0.1.16 Correct Ceph Mon Check Ports
   - 0.1.17 Skip monmap endpoint check for missing mons
+  - 0.1.18 Move ceph-mgr deployment to the ceph-mon chart
 ...