From ad09539f714715741905910801c8ba19f4449fee Mon Sep 17 00:00:00 2001 From: Stephen Taylor Date: Thu, 24 Feb 2022 14:03:45 -0700 Subject: [PATCH] [ceph-mon] Change configmap names to be based on release names This change makes the ceph-mon configmap names dynamic based on release name to match how the ceph-osd chart is naming configmaps. The new ceph-mon post-apply job needs this in some cases in order not to have conflicting configmap names in separate releases. Change-Id: Id26d0a8310ccff80a608e25d2b0a74a41f9e6a55 --- ceph-mon/Chart.yaml | 2 +- ceph-mon/templates/configmap-bin.yaml | 2 +- ceph-mon/templates/configmap-etc.yaml | 4 +- ceph-mon/templates/daemonset-mon.yaml | 24 +- .../utils/_mon_daemonset_overrides.tpl | 287 ++++++++++++++++++ ceph-mon/values.yaml | 3 + releasenotes/notes/ceph-mon.yaml | 1 + 7 files changed, 316 insertions(+), 7 deletions(-) create mode 100644 ceph-mon/templates/utils/_mon_daemonset_overrides.tpl diff --git a/ceph-mon/Chart.yaml b/ceph-mon/Chart.yaml index c22695bd5..721f52c90 100644 --- a/ceph-mon/Chart.yaml +++ b/ceph-mon/Chart.yaml @@ -15,6 +15,6 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Ceph Mon name: ceph-mon -version: 0.1.20 +version: 0.1.21 home: https://github.com/ceph/ceph ... diff --git a/ceph-mon/templates/configmap-bin.yaml b/ceph-mon/templates/configmap-bin.yaml index 59cadc10e..aede9b6af 100644 --- a/ceph-mon/templates/configmap-bin.yaml +++ b/ceph-mon/templates/configmap-bin.yaml @@ -18,7 +18,7 @@ limitations under the License. apiVersion: v1 kind: ConfigMap metadata: - name: ceph-mon-bin + name: {{ printf "%s-%s" $envAll.Release.Name "bin" | quote }} data: {{- if .Values.images.local_registry.active }} image-repo-sync.sh: | diff --git a/ceph-mon/templates/configmap-etc.yaml b/ceph-mon/templates/configmap-etc.yaml index c42e57501..b8209d3a8 100644 --- a/ceph-mon/templates/configmap-etc.yaml +++ b/ceph-mon/templates/configmap-etc.yaml @@ -12,7 +12,7 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- define "ceph.configmap.etc" }} +{{- define "ceph.mon.configmap.etc" }} {{- $configMapName := index . 0 }} {{- $envAll := index . 1 }} {{- with $envAll }} @@ -47,5 +47,5 @@ data: {{- end }} {{- end }} {{- if .Values.manifests.configmap_etc }} -{{- list "ceph-mon-etc" . | include "ceph.configmap.etc" }} +{{- list (printf "%s-%s" .Release.Name "etc") . | include "ceph.mon.configmap.etc" }} {{- end }} diff --git a/ceph-mon/templates/daemonset-mon.yaml b/ceph-mon/templates/daemonset-mon.yaml index 90043913f..a7368be01 100644 --- a/ceph-mon/templates/daemonset-mon.yaml +++ b/ceph-mon/templates/daemonset-mon.yaml @@ -15,7 +15,7 @@ limitations under the License. {{- if and .Values.manifests.daemonset_mon .Values.deployment.ceph }} {{- $envAll := . }} -{{- $serviceAccountName := "ceph-mon" }} +{{- $serviceAccountName := (printf "%s" .Release.Name) }} {{ tuple $envAll "mon" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- apiVersion: rbac.authorization.k8s.io/v1 @@ -44,6 +44,14 @@ subjects: - kind: ServiceAccount name: {{ $serviceAccountName }} namespace: {{ $envAll.Release.Namespace }} +{{- end }} + +{{- define "ceph.mon.daemonset" }} +{{- $daemonset := index . 0 }} +{{- $configMapName := index . 1 }} +{{- $serviceAccountName := index . 2 }} +{{- $envAll := index . 3 }} +{{- with $envAll }} --- kind: DaemonSet apiVersion: apps/v1 @@ -249,11 +257,11 @@ spec: emptyDir: {} - name: ceph-mon-bin configMap: - name: ceph-mon-bin + name: {{ printf "%s-%s" $envAll.Release.Name "bin" | quote }} defaultMode: 0555 - name: ceph-mon-etc configMap: - name: ceph-mon-etc + name: {{ $configMapName }} defaultMode: 0444 - name: pod-var-lib-ceph hostPath: @@ -275,3 +283,13 @@ spec: secret: secretName: {{ .Values.secrets.keyrings.mds }} {{- end }} +{{- end }} + +{{- if .Values.manifests.daemonset_mon }} +{{- $daemonset := .Values.daemonset.prefix_name }} +{{- $configMapName := (printf "%s-%s" .Release.Name "etc") }} +{{- $serviceAccountName := (printf "%s" .Release.Name) }} +{{- $daemonset_yaml := list $daemonset $configMapName $serviceAccountName . | include "ceph.mon.daemonset" | toString | fromYaml }} +{{- $configmap_yaml := "ceph.mon.configmap.etc" }} +{{- list $daemonset $daemonset_yaml $configmap_yaml $configMapName . | include "ceph.utils.mon_daemonset_overrides" }} +{{- end }} diff --git a/ceph-mon/templates/utils/_mon_daemonset_overrides.tpl b/ceph-mon/templates/utils/_mon_daemonset_overrides.tpl new file mode 100644 index 000000000..ac1998e74 --- /dev/null +++ b/ceph-mon/templates/utils/_mon_daemonset_overrides.tpl @@ -0,0 +1,287 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "ceph.utils.match_exprs_hash" }} + {{- $match_exprs := index . 0 }} + {{- $context := index . 1 }} + {{- $_ := set $context.Values "__match_exprs_hash_content" "" }} + {{- range $match_expr := $match_exprs }} + {{- $_ := set $context.Values "__match_exprs_hash_content" (print $context.Values.__match_exprs_hash_content $match_expr.key $match_expr.operator ($match_expr.values | quote)) }} + {{- end }} + {{- $context.Values.__match_exprs_hash_content | sha256sum | trunc 8 }} + {{- $_ := unset $context.Values "__match_exprs_hash_content" }} +{{- end }} + +{{- define "ceph.utils.mon_daemonset_overrides" }} + {{- $daemonset := index . 0 }} + {{- $daemonset_yaml := index . 1 }} + {{- $configmap_include := index . 2 }} + {{- $configmap_name := index . 3 }} + {{- $context := index . 4 }} + {{- $_ := unset $context ".Files" }} + {{- $_ := set $context.Values "__daemonset_yaml" $daemonset_yaml }} + {{- $daemonset_root_name := printf "ceph_%s" $daemonset }} + {{- $_ := set $context.Values "__daemonset_list" list }} + {{- $_ := set $context.Values "__default" dict }} + {{- if hasKey $context.Values.conf "overrides" }} + {{- range $key, $val := $context.Values.conf.overrides }} + + {{- if eq $key $daemonset_root_name }} + {{- range $type, $type_data := . }} + + {{- if eq $type "hosts" }} + {{- range $host_data := . }} + {{/* dictionary that will contain all info needed to generate this + iteration of the daemonset */}} + {{- $current_dict := dict }} + + {{/* set daemonset name */}} + {{- $_ := set $current_dict "name" $host_data.name }} + + {{/* apply overrides */}} + {{- $override_conf_copy := $host_data.conf }} + {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}} + {{- $root_conf_copy := omit ($context.Values.conf | toYaml | fromYaml) "overrides" }} + {{- $merged_dict := mergeOverwrite $root_conf_copy $override_conf_copy }} + {{- $root_conf_copy2 := dict "conf" $merged_dict }} + {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "conf") "__daemonset_list" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $current_dict "nodeData" $root_conf_copy4 }} + + {{/* Schedule to this host explicitly. */}} + {{- $nodeSelector_dict := dict }} + + {{- $_ := set $nodeSelector_dict "key" "kubernetes.io/hostname" }} + {{- $_ := set $nodeSelector_dict "operator" "In" }} + + {{- $values_list := list $host_data.name }} + {{- $_ := set $nodeSelector_dict "values" $values_list }} + + {{- $list_aggregate := list $nodeSelector_dict }} + {{- $_ := set $current_dict "matchExpressions" $list_aggregate }} + + {{/* store completed daemonset entry/info into global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $current_dict }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + + {{- end }} + {{- end }} + + {{- if eq $type "labels" }} + {{- $_ := set $context.Values "__label_list" . }} + {{- range $label_data := . }} + {{/* dictionary that will contain all info needed to generate this + iteration of the daemonset. */}} + {{- $_ := set $context.Values "__current_label" dict }} + + {{/* set daemonset name */}} + {{- $_ := set $context.Values.__current_label "name" $label_data.label.key }} + + {{/* apply overrides */}} + {{- $override_conf_copy := $label_data.conf }} + {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}} + {{- $root_conf_copy := omit ($context.Values.conf | toYaml | fromYaml) "overrides" }} + {{- $merged_dict := mergeOverwrite $root_conf_copy $override_conf_copy }} + {{- $root_conf_copy2 := dict "conf" $merged_dict }} + {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "conf") "__daemonset_list" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__current_label "nodeData" $root_conf_copy4 }} + + {{/* Schedule to the provided label value(s) */}} + {{- $label_dict := omit $label_data.label "NULL" }} + {{- $_ := set $label_dict "operator" "In" }} + {{- $list_aggregate := list $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + + {{/* Do not schedule to other specified labels, with higher + precedence as the list position increases. Last defined label + is highest priority. */}} + {{- $other_labels := without $context.Values.__label_list $label_data }} + {{- range $label_data2 := $other_labels }} + {{- $label_dict := omit $label_data2.label "NULL" }} + + {{- $_ := set $label_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + {{- end }} + {{- $_ := set $context.Values "__label_list" $other_labels }} + + {{/* Do not schedule to any other specified hosts */}} + {{- range $type, $type_data := $val }} + {{- if eq $type "hosts" }} + {{- range $host_data := . }} + {{- $label_dict := dict }} + + {{- $_ := set $label_dict "key" "kubernetes.io/hostname" }} + {{- $_ := set $label_dict "operator" "NotIn" }} + + {{- $values_list := list $host_data.name }} + {{- $_ := set $label_dict "values" $values_list }} + + {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{- end }} + + {{/* store completed daemonset entry/info into global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__current_label }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- $_ := unset $context.Values "__current_label" }} + + {{- end }} + {{- end }} + {{- end }} + + {{/* scheduler exceptions for the default daemonset */}} + {{- $_ := set $context.Values.__default "matchExpressions" list }} + + {{- range $type, $type_data := . }} + {{/* Do not schedule to other specified labels */}} + {{- if eq $type "labels" }} + {{- range $label_data := . }} + {{- $default_dict := omit $label_data.label "NULL" }} + + {{- $_ := set $default_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }} + {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{/* Do not schedule to other specified hosts */}} + {{- if eq $type "hosts" }} + {{- range $host_data := . }} + {{- $default_dict := dict }} + + {{- $_ := set $default_dict "key" "kubernetes.io/hostname" }} + {{- $_ := set $default_dict "operator" "NotIn" }} + + {{- $values_list := list $host_data.name }} + {{- $_ := set $default_dict "values" $values_list }} + + {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }} + {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + + {{/* generate the default daemonset */}} + + {{/* set name */}} + {{- $_ := set $context.Values.__default "name" "default" }} + + {{/* no overrides apply, so copy as-is */}} + {{- $root_conf_copy1 := omit $context.Values.conf "overrides" }} + {{- $root_conf_copy2 := dict "conf" $root_conf_copy1 }} + {{- $context_values := omit $context.Values "conf" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__default "nodeData" $root_conf_copy4 }} + + {{/* add to global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__default }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + + {{- $_ := set $context.Values "__last_configmap_name" $configmap_name }} + {{- range $current_dict := $context.Values.__daemonset_list }} + + {{- $context_novalues := omit $context "Values" }} + {{- $merged_dict := mergeOverwrite $context_novalues $current_dict.nodeData }} + {{- $_ := set $current_dict "nodeData" $merged_dict }} + + {{/* name needs to be a DNS-1123 compliant name. Ensure lower case */}} + {{- $name_format1 := printf (print $daemonset_root_name "-" $current_dict.name) | lower }} + {{/* labels may contain underscores which would be invalid here, so we replace them with dashes + there may be other valid label names which would make for an invalid DNS-1123 name + but these will be easier to handle in future with sprig regex* functions + (not availabile in helm 2.5.1) */}} + {{- $name_format2 := $name_format1 | replace "_" "-" | replace "." "-" }} + {{/* To account for the case where the same label is defined multiple times in overrides + (but with different label values), we add a sha of the scheduling data to ensure + name uniqueness */}} + {{- $_ := set $current_dict "dns_1123_name" dict }} + {{- if hasKey $current_dict "matchExpressions" }} + {{- $_ := set $current_dict "dns_1123_name" (printf (print $name_format2 "-" (list $current_dict.matchExpressions $context | include "ceph.utils.match_exprs_hash"))) }} + {{- else }} + {{- $_ := set $current_dict "dns_1123_name" $name_format2 }} + {{- end }} + + {{/* set daemonset metadata name */}} + {{- if not $context.Values.__daemonset_yaml.metadata }}{{- $_ := set $context.Values.__daemonset_yaml "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }} + + {{/* cross-reference configmap name to container volume definitions */}} + {{- $_ := set $context.Values "__volume_list" list }} + {{- range $current_volume := $context.Values.__daemonset_yaml.spec.template.spec.volumes }} + {{- $_ := set $context.Values "__volume" $current_volume }} + {{- if hasKey $context.Values.__volume "configMap" }} + {{- if eq $context.Values.__volume.configMap.name $context.Values.__last_configmap_name }} + {{- $_ := set $context.Values.__volume.configMap "name" $current_dict.dns_1123_name }} + {{- end }} + {{- end }} + {{- $updated_list := append $context.Values.__volume_list $context.Values.__volume }} + {{- $_ := set $context.Values "__volume_list" $updated_list }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "volumes" $context.Values.__volume_list }} + + {{/* populate scheduling restrictions */}} + {{- if hasKey $current_dict "matchExpressions" }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "affinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity "nodeAffinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" dict }}{{- end }} + {{- $match_exprs := dict }} + {{- $_ := set $match_exprs "matchExpressions" $current_dict.matchExpressions }} + {{- $appended_match_expr := list $match_exprs }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" $appended_match_expr }} + {{- end }} + + {{/* input value hash for current set of values overrides */}} + {{- if not $context.Values.__daemonset_yaml.spec }}{{- $_ := set $context.Values.__daemonset_yaml "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template }}{{- $_ := set $context.Values.__daemonset_yaml.spec "template" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata.annotations }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata "annotations" dict }}{{- end }} + {{- $cmap := list $current_dict.dns_1123_name $current_dict.nodeData | include $configmap_include }} + {{- $values_hash := $cmap | quote | sha256sum }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-etc-hash" $values_hash }} + + {{/* generate configmap */}} +--- +{{ $cmap }} + + {{/* generate daemonset yaml */}} +{{ range $k, $v := index $current_dict.nodeData.Values.conf.storage "mon" }} +--- +{{- $_ := set $context.Values "__tmpYAML" dict }} + +{{ $dsNodeName := index $context.Values.__daemonset_yaml.metadata "name" }} +{{ $localDsNodeName := print (trunc 54 $current_dict.dns_1123_name) "-" (print $dsNodeName $k | quote | sha256sum | trunc 8) }} +{{- if not $context.Values.__tmpYAML.metadata }}{{- $_ := set $context.Values.__tmpYAML "metadata" dict }}{{- end }} +{{- $_ := set $context.Values.__tmpYAML.metadata "name" $localDsNodeName }} + +{{ merge $context.Values.__tmpYAML $context.Values.__daemonset_yaml | toYaml }} + +{{ end }} + +--- + {{- $_ := set $context.Values "__last_configmap_name" $current_dict.dns_1123_name }} + {{- end }} +{{- end }} diff --git a/ceph-mon/values.yaml b/ceph-mon/values.yaml index 34cdc6863..e99bff93a 100644 --- a/ceph-mon/values.yaml +++ b/ceph-mon/values.yaml @@ -296,6 +296,9 @@ conf: mon: directory: /var/lib/openstack-helm/ceph/mon +daemonset: + prefix_name: "mon" + dependencies: dynamic: common: diff --git a/releasenotes/notes/ceph-mon.yaml b/releasenotes/notes/ceph-mon.yaml index 3424d6b4c..8633bf6c8 100644 --- a/releasenotes/notes/ceph-mon.yaml +++ b/releasenotes/notes/ceph-mon.yaml @@ -21,4 +21,5 @@ ceph-mon: - 0.1.18 Move ceph-mgr deployment to the ceph-mon chart - 0.1.19 Add a post-apply job to restart mons after mgrs - 0.1.20 Consolidate mon_endpoints discovery + - 0.1.21 Change configmap names to be based on release name ...