Add deep merge utility to helm-toolkit

Adds "helm-toolkit.utils.merge" which is a replacement for the
upstream sprig "merge" function which didn't quite do what we
wanted, specifically it didn't merge slices, it just overrode
one with the other.  This PS also updates existing callsites
of the sprig merge with "helm-toolkit.utils.merge".

Change-Id: I456349558d4cf941d1bcb07fc76d0688b0a10782

calico/templates/daemonset-calico-etcd.yaml

@@ -17,7 +17,8 @@ limitations under the License.
 {{- if .Values.manifests.daemonset_calico_etcd }}
 {{- $envAll := . }}
 {{- if .Values.images.local_registry.active -}}
-{{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.etcd .Values.conditional_dependencies.local_image_registry) -}}
+{{- $_ := set .Values "pod_dependency" dict -}}
+{{- $_ := include "helm-toolkit.utils.merge" (tuple .Values.pod_dependency .Values.dependencies.etcd .Values.conditional_dependencies.local_image_registry) -}}
 {{- else -}}
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.etcd -}}
 {{- end -}}

calico/templates/daemonset-calico-node.yaml

@@ -34,7 +34,8 @@ limitations under the License.
 {{- end -}}
 
 {{- if .Values.images.local_registry.active -}}
-{{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.calico_node .Values.conditional_dependencies.local_image_registry) -}}
+{{- $_ := set .Values "pod_dependency" dict -}}
+{{- $_ := include "helm-toolkit.utils.merge" (tuple .Values.pod_dependency .Values.dependencies.calico_node .Values.conditional_dependencies.local_image_registry) -}}
 {{- else -}}
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.calico_node -}}
 {{- end -}}

calico/templates/deployment-calico-kube-controllers.yaml

@@ -17,7 +17,8 @@ limitations under the License.
 {{- if .Values.manifests.deployment_calico_kube_policy_controllers }}
 {{- $envAll := . }}
 {{- if .Values.images.local_registry.active -}}
-{{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.calico_kube_policy_controllers .Values.conditional_dependencies.local_image_registry) -}}
+{{- $_ := set .Values "pod_dependency" dict -}}
+{{- $_ := include "helm-toolkit.utils.merge" (tuple .Values.pod_dependency .Values.dependencies.calico_kube_policy_controllers .Values.conditional_dependencies.local_image_registry) -}}
 {{- else -}}
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.calico_kube_policy_controllers -}}
 {{- end -}}

elasticsearch/templates/deployment-client.yaml

@@ -18,7 +18,8 @@ limitations under the License.
 {{- $envAll := . }}
 {{- $esUserSecret := .Values.secrets.elasticsearch.user }}
 {{- if .Values.images.local_registry.active -}}
-{{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.elasticsearch_client .Values.conditional_dependencies.local_image_registry) -}}
+{{- $_ := set .Values "pod_dependency" dict -}}
+{{- $_ := include "helm-toolkit.utils.merge" (tuple .Values.pod_dependency .Values.dependencies.elasticsearch_client .Values.conditional_dependencies.local_image_registry) -}}
 {{- else -}}
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.elasticsearch_client -}}
 {{- end -}}

elasticsearch/templates/deployment-master.yaml

@@ -17,7 +17,8 @@ limitations under the License.
 {{- if .Values.manifests.deployment_master }}
 {{- $envAll := . }}
 {{- if .Values.images.local_registry.active -}}
-{{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.elasticsearch_master .Values.conditional_dependencies.local_image_registry) -}}
+{{- $_ := set .Values "pod_dependency" dict -}}
+{{- $_ := include "helm-toolkit.utils.merge" (tuple .Values.pod_dependency .Values.dependencies.elasticsearch_master .Values.conditional_dependencies.local_image_registry) -}}
 {{- else -}}
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.elasticsearch_master -}}
 {{- end -}}

elasticsearch/templates/statefulset-data.yaml

@@ -17,7 +17,8 @@ limitations under the License.
 {{- if .Values.manifests.statefulset_data }}
 {{- $envAll := . }}
 {{- if .Values.images.local_registry.active -}}
-{{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.elasticsearch_data .Values.conditional_dependencies.local_image_registry) -}}
+{{- $_ := set .Values "pod_dependency" dict -}}
+{{- $_ := include "helm-toolkit.utils.merge" (tuple .Values.pod_dependency .Values.dependencies.elasticsearch_data .Values.conditional_dependencies.local_image_registry) -}}
 {{- else -}}
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.elasticsearch_data -}}
 {{- end -}}

flannel/templates/daemonset-kube-flannel-ds.yaml

@@ -17,7 +17,8 @@ limitations under the License.
 {{- if .Values.manifests.daemonset_kube_flannel_ds }}
 {{- $envAll := . }}
 {{- if .Values.images.local_registry.active -}}
-{{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.flannel .Values.conditional_dependencies.local_image_registry) -}}
+{{- $_ := set .Values "pod_dependency" dict -}}
+{{- $_ := include "helm-toolkit.utils.merge" (tuple .Values.pod_dependency .Values.dependencies.flannel .Values.conditional_dependencies.local_image_registry) -}}
 {{- else -}}
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.flannel -}}
 {{- end -}}

fluent-logging/templates/daemonset-fluent-bit.yaml

@@ -18,7 +18,8 @@ limitations under the License.
 {{- $envAll := . }}
 {{- $dependencies := .Values.dependencies.fluentbit }}
 {{- if .Values.images.local_registry.active -}}
-{{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.fluentbit .Values.conditional_dependencies.local_image_registry) -}}
+{{- $_ := set .Values "pod_dependency" dict -}}
+{{- $_ := include "helm-toolkit.utils.merge" (tuple .Values.pod_dependency .Values.dependencies.fluentbit .Values.conditional_dependencies.local_image_registry) -}}
 {{- else -}}
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.fluentbit -}}
 {{- end -}}

fluent-logging/templates/deployment-fluentd.yaml

@@ -18,7 +18,8 @@ limitations under the License.
 {{- $envAll := . }}
 {{- $esUserSecret := .Values.secrets.elasticsearch.user }}
 {{- if .Values.images.local_registry.active -}}
-{{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.fluentd .Values.conditional_dependencies.local_image_registry) -}}
+{{- $_ := set .Values "pod_dependency" dict -}}
+{{- $_ := include "helm-toolkit.utils.merge" (tuple .Values.pod_dependency .Values.dependencies.fluentd .Values.conditional_dependencies.local_image_registry) -}}
 {{- else -}}
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.fluentd -}}
 {{- end -}}

grafana/templates/deployment.yaml

@@ -17,7 +17,8 @@ limitations under the License.
 {{- if .Values.manifests.deployment }}
 {{- $envAll := . }}
 {{- if .Values.images.local_registry.active -}}
-{{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.grafana .Values.conditional_dependencies.local_image_registry) -}}
+{{- $_ := set .Values "pod_dependency" dict -}}
+{{- $_ := include "helm-toolkit.utils.merge" (tuple .Values.pod_dependency .Values.dependencies.grafana .Values.conditional_dependencies.local_image_registry) -}}
 {{- else -}}
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.grafana -}}
 {{- end -}}

helm-toolkit/templates/utils/_merge.tpl

@@ -0,0 +1,100 @@
+{{/*
+Copyright 2018 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+Takes a tuple of values and merges into the first (target) one each subsequent
+(source) one in order. If all values to merge are maps, then the tuple can be
+passed as is and the target will be the result, otherwise pass a map with a
+"values" key containing the tuple of values to merge, and the merge result will
+be assigned to the "result" key of the passed map.
+
+When merging maps, for each key in the source, if the target does not define
+that key, the source value is assigned. If both define the key, then the key
+values are merged using this algorithm (recursively) and the result is assigned
+to the target key. Slices are merged by appending them and removing any
+duplicates. Any other values are merged by simply keeping the source, and
+throwing away the target.
+*/}}
+{{- define "helm-toolkit.utils.merge" -}}
+  {{- $local := dict -}}
+  {{- if kindIs "map" $ -}}
+    {{- $_ := set $local "values" $.values -}}
+  {{- else -}}
+    {{- $_ := set $local "values" $ -}}
+  {{- end -}}
+
+  {{- $target := first $local.values -}}
+  {{- range $item := rest $local.values -}}
+    {{- $call := dict "target" $target "source" . -}}
+    {{- $_ := include "helm-toolkit.utils._merge" $call -}}
+    {{- $_ := set $local "result" $call.result -}}
+  {{- end -}}
+
+  {{- if kindIs "map" $ -}}
+    {{- $_ := set $ "result" $local.result -}}
+  {{- end -}}
+{{- end -}}
+
+{{- define "helm-toolkit.utils._merge" -}}
+  {{- $local := dict -}}
+
+  {{- $_ := set $ "result" $.source -}}
+
+  {{/*
+  TODO: Should we `fail` when trying to merge a collection (map or slice) with
+  either a different kind of collection or a scalar?
+  */}}
+
+  {{- if and (kindIs "map" $.target) (kindIs "map" $.source) -}}
+    {{- range $key, $sourceValue := $.source -}}
+      {{- if not (hasKey $.target $key) -}}
+        {{- $_ := set $local "newTargetValue" $sourceValue -}}
+        {{- if kindIs "map" $sourceValue -}}
+          {{- $copy := dict -}}
+          {{- $call := dict "target" $copy "source" $sourceValue -}}
+          {{- $_ := include "helm-toolkit.utils._merge.shallow" $call -}}
+          {{- $_ := set $local "newTargetValue" $copy -}}
+        {{- end -}}
+      {{- else -}}
+        {{- $targetValue := index $.target $key -}}
+        {{- $call := dict "target" $targetValue "source" $sourceValue -}}
+        {{- $_ := include "helm-toolkit.utils._merge" $call -}}
+        {{- $_ := set $local "newTargetValue" $call.result -}}
+      {{- end -}}
+      {{- $_ := set $.target $key $local.newTargetValue -}}
+    {{- end -}}
+    {{- $_ := set $ "result" $.target -}}
+  {{- else if and (kindIs "slice" $.target) (kindIs "slice" $.source) -}}
+    {{- $call := dict "target" $.target "source" $.source -}}
+    {{- $_ := include "helm-toolkit.utils._merge.append_slice" $call -}}
+    {{- $_ := set $ "result" (uniq $call.result) -}}
+  {{- end -}}
+{{- end -}}
+
+{{- define "helm-toolkit.utils._merge.shallow" -}}
+  {{- range $key, $value := $.source -}}
+    {{- $_ := set $.target $key $value -}}
+  {{- end -}}
+{{- end -}}
+
+{{- define "helm-toolkit.utils._merge.append_slice" -}}
+  {{- $local := dict -}}
+  {{- $_ := set $local "result" $.target -}}
+  {{- range $value := $.source -}}
+    {{- $_ := set $local "result" (append $local.result $value) -}}
+  {{- end -}}
+  {{- $_ := set $ "result" $local.result -}}
+{{- end -}}

kibana/templates/deployment.yaml

@@ -18,7 +18,8 @@ limitations under the License.
 {{- $envAll := . }}
 {{- $esUserSecret := .Values.secrets.elasticsearch.user }}
 {{- if .Values.images.local_registry.active -}}
-{{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.kibana .Values.conditional_dependencies.local_image_registry) -}}
+{{- $_ := set .Values "pod_dependency" dict -}}
+{{- $_ := include "helm-toolkit.utils.merge" (tuple .Values.pod_dependency .Values.dependencies.kibana .Values.conditional_dependencies.local_image_registry) -}}
 {{- else -}}
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.kibana -}}
 {{- end -}}

kube-dns/templates/deployment-kube-dns.yaml

@@ -17,7 +17,8 @@ limitations under the License.
 {{- if .Values.manifests.deployment_kube_dns }}
 {{- $envAll := . }}
 {{- if .Values.images.local_registry.active -}}
-{{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.kube_dns .Values.conditional_dependencies.local_image_registry) -}}
+{{- $_ := set .Values "pod_dependency" dict -}}
+{{- $_ := include "helm-toolkit.utils.merge" (tuple .Values.pod_dependency .Values.dependencies.kube_dns .Values.conditional_dependencies.local_image_registry) -}}
 {{- else -}}
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.kube_dns -}}
 {{- end -}}

nfs-provisioner/templates/deployment.yaml

@@ -17,7 +17,8 @@ limitations under the License.
 {{- if .Values.manifests.deployment }}
 {{- $envAll := . }}
 {{- if .Values.images.local_registry.active -}}
-{{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.nfs .Values.conditional_dependencies.local_image_registry) -}}
+{{- $_ := set .Values "pod_dependency" dict -}}
+{{- $_ := include "helm-toolkit.utils.merge" (tuple .Values.pod_dependency .Values.dependencies.nfs .Values.conditional_dependencies.local_image_registry) -}}
 {{- else -}}
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.nfs -}}
 {{- end -}}

prometheus-alertmanager/templates/statefulset.yaml

@@ -17,7 +17,8 @@ limitations under the License.
 {{- if .Values.manifests.statefulset }}
 {{- $envAll := . }}
 {{- if .Values.images.local_registry.active -}}
-{{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.alertmanager .Values.conditional_dependencies.local_image_registry) -}}
+{{- $_ := set .Values "pod_dependency" dict -}}
+{{- $_ := include "helm-toolkit.utils.merge" (tuple .Values.pod_dependency .Values.dependencies.alertmanager .Values.conditional_dependencies.local_image_registry) -}}
 {{- else -}}
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.alertmanager -}}
 {{- end -}}

prometheus-kube-state-metrics/templates/deployment.yaml

@@ -17,7 +17,8 @@ limitations under the License.
 {{- if .Values.manifests.deployment }}
 {{- $envAll := . }}
 {{- if .Values.images.local_registry.active -}}
-{{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.kube_state_metrics .Values.conditional_dependencies.local_image_registry) -}}
+{{- $_ := set .Values "pod_dependency" dict -}}
+{{- $_ := include "helm-toolkit.utils.merge" (tuple .Values.pod_dependency .Values.dependencies.kube_state_metrics .Values.conditional_dependencies.local_image_registry) -}}
 {{- else -}}
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.kube_state_metrics -}}
 {{- end -}}

prometheus-node-exporter/templates/daemonset.yaml

@@ -17,7 +17,8 @@ limitations under the License.
 {{- if .Values.manifests.daemonset }}
 {{- $envAll := . }}
 {{- if .Values.images.local_registry.active -}}
-{{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.node_exporter .Values.conditional_dependencies.local_image_registry) -}}
+{{- $_ := set .Values "pod_dependency" dict -}}
+{{- $_ := include "helm-toolkit.utils.merge" (tuple .Values.pod_dependency .Values.dependencies.node_exporter .Values.conditional_dependencies.local_image_registry) -}}
 {{- else -}}
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.node_exporter -}}
 {{- end -}}

prometheus-openstack-exporter/templates/deployment.yaml

@@ -18,7 +18,8 @@ limitations under the License.
 {{- $envAll := . }}
 {{- $ksUserSecret := .Values.secrets.identity.user }}
 {{- if .Values.images.local_registry.active -}}
-{{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.prometheus_openstack_exporter .Values.conditional_dependencies.local_image_registry) -}}
+{{- $_ := set .Values "pod_dependency" dict -}}
+{{- $_ := include "helm-toolkit.utils.merge" (tuple .Values.pod_dependency .Values.dependencies.prometheus_openstack_exporter .Values.conditional_dependencies.local_image_registry) -}}
 {{- else -}}
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.prometheus_openstack_exporter -}}
 {{- end -}}

prometheus/templates/statefulset.yaml

@@ -17,7 +17,8 @@ limitations under the License.
 {{- if .Values.manifests.statefulset_prometheus }}
 {{- $envAll := . }}
 {{- if .Values.images.local_registry.active -}}
-{{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.prometheus .Values.conditional_dependencies.local_image_registry) -}}
+{{- $_ := set .Values "pod_dependency" dict -}}
+{{- $_ := include "helm-toolkit.utils.merge" (tuple .Values.pod_dependency .Values.dependencies.prometheus .Values.conditional_dependencies.local_image_registry) -}}
 {{- else -}}
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.prometheus -}}
 {{- end -}}

redis/templates/deployment.yaml

@@ -17,7 +17,8 @@ limitations under the License.
 {{- if .Values.manifests.deployment }}
 {{- $envAll := . }}
 {{- if .Values.images.local_registry.active -}}
-{{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.redis .Values.conditional_dependencies.local_image_registry) -}}
+{{- $_ := set .Values "pod_dependency" dict -}}
+{{- $_ := include "helm-toolkit.utils.merge" (tuple .Values.pod_dependency .Values.dependencies.redis .Values.conditional_dependencies.local_image_registry) -}}
 {{- else -}}
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.redis -}}
 {{- end -}}

registry/templates/daemonset-registry-proxy.yaml

@@ -17,7 +17,8 @@ limitations under the License.
 {{- if .Values.manifests.daemonset_registry_proxy }}
 {{- $envAll := . }}
 {{- if .Values.images.local_registry.active -}}
-{{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.registry_proxy .Values.conditional_dependencies.local_image_registry) -}}
+{{- $_ := set .Values "pod_dependency" dict -}}
+{{- $_ := include "helm-toolkit.utils.merge" (tuple .Values.pod_dependency .Values.dependencies.registry_proxy .Values.conditional_dependencies.local_image_registry) -}}
 {{- else -}}
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.registry_proxy -}}
 {{- end -}}

registry/templates/deployment-registry.yaml

@@ -17,7 +17,8 @@ limitations under the License.
 {{- if .Values.manifests.deployment_registry }}
 {{- $envAll := . }}
 {{- if .Values.images.local_registry.active -}}
-{{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.registry .Values.conditional_dependencies.local_image_registry) -}}
+{{- $_ := set .Values "pod_dependency" dict -}}
+{{- $_ := include "helm-toolkit.utils.merge" (tuple .Values.pod_dependency .Values.dependencies.registry .Values.conditional_dependencies.local_image_registry) -}}
 {{- else -}}
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.registry -}}
 {{- end -}}

registry/templates/job-bootstrap.yaml

@@ -18,7 +18,8 @@ limitations under the License.
 {{- $envAll := . }}
 {{- if .Values.bootstrap.enabled }}
 {{- if .Values.images.local_registry.active -}}
-{{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.bootstrap .Values.conditional_dependencies.local_image_registry) -}}
+{{- $_ := set .Values "pod_dependency" dict -}}
+{{- $_ := include "helm-toolkit.utils.merge" (tuple .Values.pod_dependency .Values.dependencies.bootstrap .Values.conditional_dependencies.local_image_registry) -}}
 {{- else -}}
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.bootstrap -}}
 {{- end -}}

tiller/templates/deployment-tiller.yaml

@@ -17,7 +17,8 @@ limitations under the License.
 {{- if .Values.manifests.deployment_tiller }}
 {{- $envAll := . }}
 {{- if .Values.images.local_registry.active -}}
-{{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.tiller .Values.conditional_dependencies.local_image_registry) -}}
+{{- $_ := set .Values "pod_dependency" dict -}}
+{{- $_ := include "helm-toolkit.utils.merge" (tuple .Values.pod_dependency .Values.dependencies.tiller .Values.conditional_dependencies.local_image_registry) -}}
 {{- else -}}
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.tiller -}}
 {{- end -}}