openstack/openstack-helm-infra
Code Issues Proposed changes

Move ceph-mgr deployment to the ceph-mon chart

Browse Source 
This change moves the ceph-mgr deployment from the ceph-client
chart to the ceph-mon chart. Its purpose is to facilitate the
proper Ceph upgrade procedure, which prescribes restarting mgr
daemons before mon daemons.

There will be additional work required to implement the correct
daemon restart procedure for upgrades. This change only addresses
the move of the ceph-mgr deployment.

Change-Id: I3ac4a75f776760425c88a0ba1edae5fb339f128d
This commit is contained in:
Stephen Taylor 2022-02-02 13:03:58 -07:00
parent f69ea0ea86
commit ea2c0115c4
17 changed files with 199 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ceph-client/Chart.yaml
View File

@ -15,6 +15,6 @@ apiVersion: v1
appVersion: v1.0.0
description: OpenStack-Helm Ceph Client
name: ceph-client
version: 0.1.29
version: 0.1.30
home: https://github.com/ceph/ceph-client
...

7
ceph-client/templates/configmap-bin.yaml
View File

@ -43,11 +43,6 @@ data:
mds-start.sh: |
{{ tuple "bin/mds/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
mgr-start.sh: |
{{ tuple "bin/mgr/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
mgr-check.sh: |
{{ tuple "bin/mgr/_check.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
helm-tests.sh: |
{{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
utils-checkDNS.sh: |
@ -55,8 +50,6 @@ data:
utils-checkDNS_start.sh: |
{{ tuple "bin/utils/_checkDNS_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
utils-checkPGs.py: |
{{ tuple "bin/utils/_checkPGs.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
utils-checkPGs.sh: |
{{ tuple "bin/utils/_checkPGs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

4
ceph-client/templates/cronjob-checkPGs.yaml
View File

@ -91,10 +91,6 @@ spec:
mountPath: /tmp
- name: pod-etc-ceph
mountPath: /etc/ceph
- name: ceph-client-bin
mountPath: /tmp/utils-checkPGs.py
subPath: utils-checkPGs.py
readOnly: true
- name: ceph-client-bin
mountPath: /tmp/utils-checkPGs.sh
subPath: utils-checkPGs.sh

88
ceph-client/values.yaml
View File

@ -27,7 +27,6 @@ images:
ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:change_770201_ubuntu_bionic-20210113'
ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:change_770201_ubuntu_bionic-20210113'
ceph_mds: 'docker.io/openstackhelm/ceph-daemon:change_770201_ubuntu_bionic-20210113'
ceph_mgr: 'docker.io/openstackhelm/ceph-daemon:change_770201_ubuntu_bionic-20210113'
ceph_rbd_pool: 'docker.io/openstackhelm/ceph-config-helper:change_770201_ubuntu_bionic-20210113'
dep_check: 'quay.io/airshipit/kubernetes-entrypoint:v1.0.0'
image_repo_sync: 'docker.io/library/docker:17.07.0'
@ -44,12 +43,12 @@ labels:
test:
node_selector_key: openstack-control-plane
node_selector_value: enabled
mds:
node_selector_key: ceph-mds
node_selector_value: enabled
mgr:
node_selector_key: ceph-mgr
node_selector_value: enabled
mds:
node_selector_key: ceph-mds
node_selector_value: enabled
checkdns:
node_selector_key: ceph-mon
node_selector_value: enabled
@ -74,17 +73,6 @@ pod:
runAsUser: 64045
readOnlyRootFilesystem: true
allowPrivilegeEscalation: false
mgr:
pod:
runAsUser: 65534
container:
init_dirs:
runAsUser: 0
readOnlyRootFilesystem: true
mgr:
runAsUser: 64045
readOnlyRootFilesystem: true
allowPrivilegeEscalation: false
bootstrap:
pod:
runAsUser: 65534
@ -109,7 +97,6 @@ pod:
dns_policy: "ClusterFirstWithHostNet"
replicas:
mds: 2
mgr: 2
lifecycle:
upgrades:
deployments:
@ -118,9 +105,6 @@ pod:
rolling_update:
max_surge: 25%
max_unavailable: 25%
updateStrategy:
mgr:
type: Recreate
affinity:
anti:
type:
@ -138,13 +122,6 @@ pod:
limits:
memory: "50Mi"
cpu: "500m"
mgr:
requests:
memory: "5Mi"
cpu: "250m"
limits:
memory: "50Mi"
cpu: "500m"
checkdns:
requests:
memory: "5Mi"
@ -202,16 +179,6 @@ pod:
key: node.kubernetes.io/unreachable
operator: Exists
tolerationSeconds: 60
mgr:
tolerations:
- effect: NoExecute
key: node.kubernetes.io/not-ready
operator: Exists
tolerationSeconds: 60
- effect: NoExecute
key: node.kubernetes.io/unreachable
operator: Exists
tolerationSeconds: 60
secrets:
keyrings:
@ -259,7 +226,6 @@ jobs:
conf:
features:
mds: true
mgr: true
pg_autoscaler: true
cluster_flags:
# List of flags to set or unset separated by spaces
@ -489,13 +455,6 @@ dependencies:
services:
- endpoint: internal
service: ceph_mon
mgr:
jobs:
- ceph-storage-keys-generator
- ceph-mgr-keyring-generator
services:
- endpoint: internal
service: ceph_mon
pool_checkpgs:
jobs:
- ceph-rbd-pool
@ -542,38 +501,6 @@ bootstrap:
}
#ensure_pool volumes 8 cinder
# Uncomment below to enable mgr modules
# For a list of available modules:
# http://docs.ceph.com/docs/master/mgr/
# This overrides mgr_initial_modules (default: restful, status)
# Any module not listed here will be disabled
ceph_mgr_enabled_modules:
- restful
- status
- prometheus
- balancer
- iostat
- pg_autoscaler
# You can configure your mgr modules
# below. Each module has its own set
# of key/value. Refer to the doc
# above for more info. For example:
ceph_mgr_modules_config:
# balancer:
# active: 1
# prometheus:
# server_port: 9283
# server_addr: 0.0.0.0
# dashboard:
# port: 7000
# localpool:
# failure_domain: host
# subtree: rack
# pg_num: "128"
# num_rep: "3"
# min_size: "2"
endpoints:
cluster_domain_suffix: cluster.local
local_image_registry:
@ -614,26 +541,17 @@ endpoints:
scheme:
default: http
monitoring:
prometheus:
enabled: true
ceph_mgr:
scrape: true
port: 9283
manifests:
configmap_bin: true
configmap_test_bin: true
configmap_etc: true
deployment_mds: true
deployment_mgr: true
deployment_checkdns: true
job_bootstrap: false
job_ns_client_ceph_config: true
job_cephfs_client_key: true
job_image_repo_sync: true
job_rbd_pool: true
service_mgr: true
helm_tests: true
cronjob_checkPGs: true
cronjob_defragosds: true

3
ceph-client/values_overrides/apparmor.yaml
View File

@ -8,9 +8,6 @@ pod:
ceph-mds:
ceph-mds: runtime/default
ceph-init-dirs: runtime/default
ceph-mgr:
ceph-mgr: runtime/default
ceph-init-dirs: runtime/default
ceph-rbd-pool:
ceph-rbd-pool: runtime/default
init: runtime/default

2
ceph-mon/Chart.yaml
View File

@ -15,6 +15,6 @@ apiVersion: v1
appVersion: v1.0.0
description: OpenStack-Helm Ceph Mon
name: ceph-mon
version: 0.1.17
version: 0.1.18
home: https://github.com/ceph/ceph
...

0
ceph-client/templates/bin/mgr/_check.sh.tpl → ceph-mon/templates/bin/mgr/_check.sh.tpl
View File

2
ceph-client/templates/bin/mgr/_start.sh.tpl → ceph-mon/templates/bin/mgr/_start.sh.tpl
View File

@ -6,7 +6,7 @@ set -ex
: "${ADMIN_KEYRING:=/etc/ceph/${CLUSTER}.client.admin.keyring}"
: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}"
{{ include "ceph-client.snippets.mon_host_from_k8s_ep" . }}
{{ include "ceph-mon.snippets.mon_host_from_k8s_ep" . }}
if [[ ! -e ${CEPH_CONF}.template ]]; then
echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon"

0
ceph-client/templates/bin/utils/_checkPGs.py.tpl → ceph-mon/templates/bin/utils/_checkPGs.py.tpl
View File

8
ceph-mon/templates/configmap-bin.yaml
View File

@ -47,6 +47,11 @@ data:
mon-check.sh: |
{{ tuple "bin/mon/_check.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
mgr-start.sh: |
{{ tuple "bin/mgr/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
mgr-check.sh: |
{{ tuple "bin/mgr/_check.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
moncheck-start.sh: |
{{ tuple "bin/moncheck/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
moncheck-reap-zombies.py: |
@ -57,3 +62,6 @@ data:
utils-checkDNS.sh: |
{{ tuple "bin/utils/_checkDNS.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- end }}
utils-checkPGs.py: |
{{ tuple "bin/utils/_checkPGs.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

34
ceph-client/templates/deployment-mgr.yaml → ceph-mon/templates/deployment-mgr.yaml
View File

@ -40,7 +40,7 @@ spec:
annotations:
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
configmap-etc-client-hash: {{ tuple "configmap-etc-client.yaml" . | include "helm-toolkit.utils.hash" }}
configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
{{ dict "envAll" $envAll "podName" "ceph-mgr" "containerNames" (list "ceph-mgr" "ceph-init-dirs") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
spec:
{{ dict "envAll" $envAll "application" "mgr" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
@ -56,7 +56,7 @@ spec:
initContainers:
{{ tuple $envAll "mgr" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
- name: ceph-init-dirs
{{ tuple $envAll "ceph_mds" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll "ceph_mgr" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ dict "envAll" $envAll "application" "mgr" "container" "init_dirs" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
command:
- /tmp/init-dirs.sh
@ -70,7 +70,7 @@ spec:
mountPath: /run
- name: pod-etc-ceph
mountPath: /etc/ceph
- name: ceph-client-bin
- name: ceph-mon-bin
mountPath: /tmp/init-dirs.sh
subPath: init-dirs.sh
readOnly: true
@ -142,23 +142,19 @@ spec:
mountPath: /run
- name: pod-etc-ceph
mountPath: /etc/ceph
- name: ceph-client-bin
- name: ceph-mon-bin
mountPath: /mgr-start.sh
subPath: mgr-start.sh
readOnly: true
- name: ceph-client-bin
- name: ceph-mon-bin
mountPath: /tmp/mgr-check.sh
subPath: mgr-check.sh
readOnly: true
- name: ceph-client-bin
mountPath: /tmp/utils-checkDNS.sh
subPath: utils-checkDNS.sh
readOnly: true
- name: ceph-client-etc
- name: ceph-mon-etc
mountPath: /etc/ceph/ceph.conf.template
subPath: ceph.conf
readOnly: true
- name: ceph-client-admin-keyring
- name: ceph-mon-admin-keyring
mountPath: /etc/ceph/ceph.client.admin.keyring
subPath: ceph.client.admin.keyring
readOnly: true
@ -172,14 +168,10 @@ spec:
- name: pod-var-lib-ceph-crash
mountPath: /var/lib/ceph/crash
readOnly: false
- name: ceph-client-bin
- name: ceph-mon-bin
mountPath: /tmp/utils-checkPGs.py
subPath: utils-checkPGs.py
readOnly: true
- name: ceph-client-bin
mountPath: /tmp/utils-checkPGs.sh
subPath: utils-checkPGs.sh
readOnly: true
volumes:
- name: pod-tmp
emptyDir: {}
@ -188,13 +180,13 @@ spec:
medium: "Memory"
- name: pod-etc-ceph
emptyDir: {}
- name: ceph-client-bin
- name: ceph-mon-bin
configMap:
name: ceph-client-bin
name: ceph-mon-bin
defaultMode: 0555
- name: ceph-client-etc
- name: ceph-mon-etc
configMap:
name: ceph-client-etc
name: ceph-mon-etc
defaultMode: 0444
- name: pod-var-lib-ceph
emptyDir: {}
@ -202,7 +194,7 @@ spec:
hostPath:
path: /var/lib/openstack-helm/ceph/crash
type: DirectoryOrCreate
- name: ceph-client-admin-keyring
- name: ceph-mon-admin-keyring
secret:
secretName: {{ .Values.secrets.keyrings.admin }}
- name: ceph-bootstrap-mgr-keyring

0
ceph-client/templates/service-mgr.yaml → ceph-mon/templates/service-mgr.yaml
View File

68
ceph-mon/templates/snippets/_mon_host_from_k8s_ep.sh.tpl Normal file
View File

@ -0,0 +1,68 @@
{{- define "ceph-mon.snippets.mon_host_from_k8s_ep" -}}
{{/*
Inserts a bash function definition mon_host_from_k8s_ep() which can be used
to construct a mon_hosts value from the given namespaced endpoint.
Usage (e.g. in _script.sh.tpl):
#!/bin/bash
: "${NS:=ceph}"
: "${EP:=ceph-mon-discovery}"
{{ include "ceph-mon.snippets.mon_host_from_k8s_ep" . }}
MON_HOST=$(mon_host_from_k8s_ep "$NS" "$EP")
if [ -z "$MON_HOST" ]; then
# deal with failure
else
sed -i -e "s/^mon_host = /mon_host = $MON_HOST/" /etc/ceph/ceph.conf
fi
*/}}
{{`
# Construct a mon_hosts value from the given namespaced endpoint
# IP x.x.x.x with port p named "mon-msgr2" will appear as [v2:x.x.x.x/p/0]
# IP x.x.x.x with port q named "mon" will appear as [v1:x.x.x.x/q/0]
# IP x.x.x.x with ports p and q will appear as [v2:x.x.x.x/p/0,v1:x.x.x.x/q/0]
# The entries for all IPs will be joined with commas
mon_host_from_k8s_ep() {
local ns=$1
local ep=$2
if [ -z "$ns" ] || [ -z "$ep" ]; then
return 1
fi
# We don't want shell expansion for the go-template expression
# shellcheck disable=SC2016
kubectl get endpoints -n "$ns" "$ep" -o go-template='
{{- $sep := "" }}
{{- range $_,$s := .subsets }}
{{- $v2port := 0 }}
{{- $v1port := 0 }}
{{- range $_,$port := index $s "ports" }}
{{- if (eq $port.name "mon-msgr2") }}
{{- $v2port = $port.port }}
{{- else if (eq $port.name "mon") }}
{{- $v1port = $port.port }}
{{- end }}
{{- end }}
{{- range $_,$address := index $s "addresses" }}
{{- $v2endpoint := printf "v2:%s:%d/0" $address.ip $v2port }}
{{- $v1endpoint := printf "v1:%s:%d/0" $address.ip $v1port }}
{{- if (and $v2port $v1port) }}
{{- printf "%s[%s,%s]" $sep $v2endpoint $v1endpoint }}
{{- $sep = "," }}
{{- else if $v2port }}
{{- printf "%s[%s]" $sep $v2endpoint }}
{{- $sep = "," }}
{{- else if $v1port }}
{{- printf "%s[%s]" $sep $v1endpoint }}
{{- $sep = "," }}
{{- end }}
{{- end }}
{{- end }}'
}
`}}
{{- end -}}

99
ceph-mon/values.yaml
View File

@ -26,6 +26,7 @@ images:
ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:change_770201_ubuntu_bionic-20210113'
ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:change_770201_ubuntu_bionic-20210113'
ceph_mon: 'docker.io/openstackhelm/ceph-daemon:change_770201_ubuntu_bionic-20210113'
ceph_mgr: 'docker.io/openstackhelm/ceph-daemon:change_770201_ubuntu_bionic-20210113'
ceph_mon_check: 'docker.io/openstackhelm/ceph-config-helper:change_770201_ubuntu_bionic-20210113'
dep_check: 'quay.io/airshipit/kubernetes-entrypoint:v1.0.0'
image_repo_sync: 'docker.io/library/docker:17.07.0'
@ -42,6 +43,9 @@ labels:
mon:
node_selector_key: ceph-mon
node_selector_value: enabled
mgr:
node_selector_key: ceph-mgr
node_selector_value: enabled
pod:
security_context:
@ -59,6 +63,17 @@ pod:
runAsUser: 64045
readOnlyRootFilesystem: true
allowPrivilegeEscalation: false
mgr:
pod:
runAsUser: 65534
container:
init_dirs:
runAsUser: 0
readOnlyRootFilesystem: true
mgr:
runAsUser: 64045
readOnlyRootFilesystem: true
allowPrivilegeEscalation: false
moncheck:
pod:
runAsUser: 65534
@ -98,6 +113,7 @@ pod:
readOnlyRootFilesystem: true
dns_policy: "ClusterFirstWithHostNet"
replicas:
mgr: 2
mon_check: 1
lifecycle:
upgrades:
@ -107,6 +123,9 @@ pod:
enabled: true
min_ready_seconds: 0
max_unavailable: 1
updateStrategy:
mgr:
type: Recreate
affinity:
anti:
type:
@ -124,6 +143,13 @@ pod:
limits:
memory: "100Mi"
cpu: "500m"
mgr:
requests:
memory: "5Mi"
cpu: "250m"
limits:
memory: "50Mi"
cpu: "500m"
mon_check:
requests:
memory: "5Mi"
@ -154,6 +180,16 @@ pod:
memory: "1024Mi"
cpu: "2000m"
tolerations:
mgr:
tolerations:
- effect: NoExecute
key: node.kubernetes.io/not-ready
operator: Exists
tolerationSeconds: 60
- effect: NoExecute
key: node.kubernetes.io/unreachable
operator: Exists
tolerationSeconds: 60
mon_check:
tolerations:
- effect: NoExecute
@ -178,6 +214,8 @@ network:
cluster: 192.168.0.0/16
conf:
features:
mgr: true
templates:
keyring:
admin: |
@ -272,6 +310,13 @@ dependencies:
jobs:
- ceph-storage-keys-generator
- ceph-mon-keyring-generator
mgr:
jobs:
- ceph-storage-keys-generator
- ceph-mgr-keyring-generator
services:
- endpoint: internal
service: ceph_mon
moncheck:
jobs:
- ceph-storage-keys-generator
@ -298,6 +343,38 @@ bootstrap:
}
#ensure_pool volumes 8 cinder
# Uncomment below to enable mgr modules
# For a list of available modules:
# http://docs.ceph.com/docs/master/mgr/
# This overrides mgr_initial_modules (default: restful, status)
# Any module not listed here will be disabled
ceph_mgr_enabled_modules:
- restful
- status
- prometheus
- balancer
- iostat
- pg_autoscaler
# You can configure your mgr modules
# below. Each module has its own set
# of key/value. Refer to the doc
# above for more info. For example:
ceph_mgr_modules_config:
# balancer:
# active: 1
# prometheus:
# server_port: 9283
# server_addr: 0.0.0.0
# dashboard:
# port: 7000
# localpool:
# failure_domain: host
# subtree: rack
# pg_num: "128"
# num_rep: "3"
# min_size: "2"
# if you change provision_storage_class to false
# it is presumed you manage your own storage
# class definition externally
@ -344,17 +421,39 @@ endpoints:
default: 6789
mon_msgr2:
default: 3300
ceph_mgr:
namespace: null
hosts:
default: ceph-mgr
host_fqdn_override:
default: null
port:
mgr:
default: 7000
metrics:
default: 9283
scheme:
default: http
monitoring:
prometheus:
enabled: true
ceph_mgr:
scrape: true
port: 9283
manifests:
configmap_bin: true
configmap_etc: true
configmap_templates: true
daemonset_mon: true
deployment_mgr: true
deployment_moncheck: true
job_image_repo_sync: true
job_bootstrap: true
job_keyring: true
service_mon: true
service_mgr: true
service_mon_discovery: true
job_storage_admin_keys: true
...

3
ceph-mon/values_overrides/apparmor.yaml
View File

@ -6,6 +6,9 @@ pod:
ceph-init-dirs: runtime/default
ceph-mon: runtime/default
ceph-log-ownership: runtime/default
ceph-mgr:
ceph-mgr: runtime/default
ceph-init-dirs: runtime/default
ceph-mon-check:
ceph-mon: runtime/default
init: runtime/default

1
releasenotes/notes/ceph-client.yaml
View File

@ -30,4 +30,5 @@ ceph-client:
- 0.1.27 Update ceph_mon config to ips from fqdn
- 0.1.28 Fix ceph.conf update job labels, rendering
- 0.1.29 Consolidate mon_host discovery
- 0.1.30 Move ceph-mgr deployment to the ceph-mon chart
...

1
releasenotes/notes/ceph-mon.yaml
View File

@ -18,4 +18,5 @@ ceph-mon:
- 0.1.15 Prevent mon-check from removing mons when down temporarily
- 0.1.16 Correct Ceph Mon Check Ports
- 0.1.17 Skip monmap endpoint check for missing mons
- 0.1.18 Move ceph-mgr deployment to the ceph-mon chart
...