diff --git a/manifests/function/k8scontrol/controlplane.yaml b/manifests/function/k8scontrol/controlplane.yaml index 72b6446d7..8b23d4a7e 100644 --- a/manifests/function/k8scontrol/controlplane.yaml +++ b/manifests/function/k8scontrol/controlplane.yaml @@ -13,6 +13,13 @@ spec: clusterConfiguration: apiServer: timeoutForControlPlane: 1000s + files: + - path: "/etc/systemd/system/docker.service.d/http-proxy.conf" + content: | + [Service] + Environment="HTTP_PROXY=REPLACEMENT_HTTP_PROXY" + Environment="HTTPS_PROXY=REPLACEMENT_HTTPS_PROXY" + Environment="NO_PROXY=REPLACEMENT_NO_PROXY" preKubeadmCommands: - echo 'root:r00tme' | chpasswd - echo 'ubuntu:r00tme' | chpasswd @@ -23,6 +30,12 @@ spec: EOF - sysctl --system - swapoff -a + - export HTTP_PROXY=REPLACEMENT_HTTP_PROXY + - export HTTPS_PROXY=REPLACEMENT_HTTPS_PROXY + - export http_proxy=${HTTP_PROXY} + - export https_proxy=${HTTPS_PROXY} + - export NO_PROXY=REPLACEMENT_NO_PROXY + - export no_proxy=${NO_PROXY} - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - echo "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | tee -a /etc/apt/sources.list @@ -35,6 +48,7 @@ spec: containerd.io - apt install -y kubelet=1.18.6-00 kubeadm=1.18.6-00 kubectl=1.18.6-00 - apt-mark hold docker-ce docker-ce-cli containerd.io kubelet kubeadm kubectl + - unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY no_proxy NO_PROXY initConfiguration: nodeRegistration: name: '{{ ds.meta_data.local_hostname }}' diff --git a/manifests/function/k8scontrol/replacements/k8scontrol-env-vars.yaml b/manifests/function/k8scontrol/replacements/k8scontrol-env-vars.yaml new file mode 100644 index 000000000..0f5a53e65 --- /dev/null +++ b/manifests/function/k8scontrol/replacements/k8scontrol-env-vars.yaml @@ -0,0 +1,40 @@ +# These rules inject env vars into the k8scontrol function. +apiVersion: airshipit.org/v1alpha1 +kind: ReplacementTransformer +metadata: + name: k8scontrol-env-vars-replacements +replacements: +# Replace the proxy vars +- source: + objref: + name: env-vars-catalogue + fieldref: env.HTTP_PROXY + target: + objref: + kind: KubeadmControlPlane + name: cluster-controlplane + fieldrefs: + - "spec.kubeadmConfigSpec.preKubeadmCommands%REPLACEMENT_HTTP_PROXY%" + - "spec.kubeadmConfigSpec.files[path=/etc/systemd/system/docker.service.d/http-proxy.conf].content%REPLACEMENT_HTTP_PROXY%" +- source: + objref: + name: env-vars-catalogue + fieldref: env.HTTPS_PROXY + target: + objref: + kind: KubeadmControlPlane + name: cluster-controlplane + fieldrefs: + - "spec.kubeadmConfigSpec.preKubeadmCommands%REPLACEMENT_HTTPS_PROXY%" + - "spec.kubeadmConfigSpec.files[path=/etc/systemd/system/docker.service.d/http-proxy.conf].content%REPLACEMENT_HTTPS_PROXY%" +- source: + objref: + name: env-vars-catalogue + fieldref: env.NO_PROXY + target: + objref: + kind: KubeadmControlPlane + name: cluster-controlplane + fieldrefs: + - "spec.kubeadmConfigSpec.preKubeadmCommands%REPLACEMENT_NO_PROXY%" + - "spec.kubeadmConfigSpec.files[path=/etc/systemd/system/docker.service.d/http-proxy.conf].content%REPLACEMENT_NO_PROXY%" diff --git a/manifests/function/k8scontrol/replacements/kustomization.yaml b/manifests/function/k8scontrol/replacements/kustomization.yaml index 1d43ee154..9006bf484 100644 --- a/manifests/function/k8scontrol/replacements/kustomization.yaml +++ b/manifests/function/k8scontrol/replacements/kustomization.yaml @@ -2,3 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - versions.yaml + - k8scontrol-env-vars.yaml diff --git a/pkg/document/plugin/replacement/transformer.go b/pkg/document/plugin/replacement/transformer.go index 74e7f69a3..b5db9b8ed 100644 --- a/pkg/document/plugin/replacement/transformer.go +++ b/pkg/document/plugin/replacement/transformer.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "io/ioutil" + "reflect" "regexp" "strconv" "strings" @@ -209,12 +210,6 @@ func substituteSubstring(tgt *yaml.RNode, fieldRef, substringPattern string, val switch curVal.YNode().Kind { case yaml.ScalarNode: p := regexp.MustCompile(substringPattern) - if !p.MatchString(yaml.GetValue(curVal)) { - return ErrPatternSubstring{ - Msg: fmt.Sprintf("pattern '%s' is defined in configuration but was not found in target value %s", - substringPattern, yaml.GetValue(curVal)), - } - } curVal.YNode().Value = p.ReplaceAllString(yaml.GetValue(curVal), yaml.GetValue(value)) case yaml.SequenceNode: @@ -227,12 +222,6 @@ func substituteSubstring(tgt *yaml.RNode, fieldRef, substringPattern string, val return err } p := regexp.MustCompile(substringPattern) - if !p.MatchString(yaml.GetValue(item)) { - return ErrPatternSubstring{ - Msg: fmt.Sprintf("pattern '%s' is defined in configuration but was not found in target value %s", - substringPattern, yaml.GetValue(item)), - } - } item.YNode().Value = p.ReplaceAllString(yaml.GetValue(item), yaml.GetValue(value)) } default: @@ -343,6 +332,32 @@ func extractSubstringPattern(path string) (extractedPath string, substringPatter return groups[1], groups[2] } +// replaces substring in a string if pattern applies +func processString(field string, substringPattern string, replacement string) string { + pattern := regexp.MustCompile(substringPattern) + return pattern.ReplaceAllString(field, replacement) +} + +// replaces substring in any string in the array if pattern applies +func processArray(tgt []string, pattern string, replacement string) []string { + result := make([]string, 0, len(tgt)) + for _, field := range tgt { + result = append(result, processString(field, pattern, replacement)) + } + return result +} + +// converts array of interfaces to array of strings +func convertToStrings(iFaces []interface{}) ([]string, bool) { + result := []string{} + for _, val := range iFaces { + if str, ok := val.(string); ok { + result = append(result, str) + } + } + return result, (len(result) != 0) +} + // apply a substring substitution based on a pattern func applySubstringPattern(target interface{}, replacement interface{}, substringPattern string) (regexedReplacement interface{}, err error) { @@ -364,19 +379,23 @@ func applySubstringPattern(target interface{}, replacement interface{}, "with string or numeric replacement values"} } - tgt, ok := target.(string) - if !ok { - return nil, ErrPatternSubstring{Msg: "pattern-based substitution can only be applied to string target fields"} - } - - p := regexp.MustCompile(substringPattern) - if !p.MatchString(tgt) { - return nil, ErrPatternSubstring{ - Msg: fmt.Sprintf("pattern '%s' is defined in configuration but was not found in target value %s", - substringPattern, tgt), + switch reflect.TypeOf(target).Kind() { + case reflect.String: + return processString(target.(string), substringPattern, replacementString), nil + case reflect.Slice: + if ifaceArray, ok := target.([]interface{}); ok { + if strArray, ok := convertToStrings(ifaceArray); ok { + return processArray(strArray, substringPattern, replacementString), nil + } + } + if strArray, ok := target.([]string); ok { + return processArray(strArray, substringPattern, replacementString), nil } } - return p.ReplaceAllString(tgt, replacementString), nil + return nil, ErrPatternSubstring{ + Msg: "pattern-based substitution can only be applied to string " + + "or array of strings target fields", + } } func updateMapField(m map[string]interface{}, pathToField []string, replacement interface{}) error { diff --git a/pkg/document/plugin/replacement/transformer_test.go b/pkg/document/plugin/replacement/transformer_test.go index d7996809a..1dd2b1e79 100644 --- a/pkg/document/plugin/replacement/transformer_test.go +++ b/pkg/document/plugin/replacement/transformer_test.go @@ -83,7 +83,7 @@ var testCases = []struct { apiVersion: airshipit.org/v1alpha1 kind: ReplacementTransformer metadata: - name: notImportantHere + name: Test_Case_0 replacements: - source: value: nginx:newtag @@ -158,7 +158,7 @@ spec: apiVersion: airshipit.org/v1alpha1 kind: ReplacementTransformer metadata: - name: notImportantHere + name: Test_Case_1 replacements: - source: value: 1.17.0 @@ -195,13 +195,82 @@ spec: name: nginx-tagged `, }, - { cfg: ` apiVersion: airshipit.org/v1alpha1 kind: ReplacementTransformer metadata: - name: notImportantHere + name: Test_Case_2 +replacements: +- source: + value: test.proxy.com + target: + objref: + kind: Secret + name: deploy1_secret + fieldrefs: + - stringData%REPLACEME% +`, + + in: ` +apiVersion: v1 +kind: Secret +metadata: + name: deploy1_secret +stringData: PROXY=REPLACEME +type: Opaque +`, + expectedOut: `apiVersion: v1 +kind: Secret +metadata: + name: deploy1_secret +stringData: PROXY=test.proxy.com +type: Opaque +`, + }, + { + cfg: ` +apiVersion: airshipit.org/v1alpha1 +kind: ReplacementTransformer +metadata: + name: Test_Case_3 +replacements: +- source: + value: testString + target: + objref: + kind: KubeadmControlPlane + name: cluster-controlplane + fieldrefs: + - spec.kubeadmConfigSpec.preKubeadmCommands%REPLACEME% +`, + + in: ` +apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 +kind: KubeadmControlPlane +metadata: + name: cluster-controlplane +spec: + kubeadmConfigSpec: + preKubeadmCommands: + - echo REPLACEME +`, + expectedOut: `apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 +kind: KubeadmControlPlane +metadata: + name: cluster-controlplane +spec: + kubeadmConfigSpec: + preKubeadmCommands: + - echo testString +`, + }, + { + cfg: ` +apiVersion: airshipit.org/v1alpha1 +kind: ReplacementTransformer +metadata: + name: Test_Case_4 replacements: - source: objref: @@ -270,7 +339,7 @@ spec: apiVersion: airshipit.org/v1alpha1 kind: ReplacementTransformer metadata: - name: notImportantHere + name: Test_Case_5 replacements: - source: objref: @@ -370,7 +439,7 @@ metadata: apiVersion: airshipit.org/v1alpha1 kind: ReplacementTransformer metadata: - name: notImportantHere + name: Test_Case_6 replacements: - source: value: regexedtag @@ -442,7 +511,7 @@ spec: apiVersion: airshipit.org/v1alpha1 kind: ReplacementTransformer metadata: - name: notImportantHere + name: Test_Case_7 replacements: - source: objref: @@ -499,7 +568,7 @@ spec: apiVersion: airshipit.org/v1alpha1 kind: ReplacementTransformer metadata: - name: notImportantHere + name: Test_Case_8 replacements: - source: objref: @@ -558,7 +627,7 @@ spec: apiVersion: airshipit.org/v1alpha1 kind: ReplacementTransformer metadata: - name: test-for-numeric-conversion + name: Test_Case_9 replacements: - source: objref: @@ -617,7 +686,7 @@ spec: apiVersion: airshipit.org/v1alpha1 kind: ReplacementTransformer metadata: - name: notImportantHere + name: Test_Case_10 replacements: - source: objref: @@ -652,7 +721,7 @@ spec: apiVersion: airshipit.org/v1alpha1 kind: ReplacementTransformer metadata: - name: notImportantHere + name: Test_Case_11 replacements: - source: objref: @@ -676,7 +745,7 @@ metadata: apiVersion: airshipit.org/v1alpha1 kind: ReplacementTransformer metadata: - name: notImportantHere + name: Test_Case_12 replacements: - source: objref: @@ -703,7 +772,7 @@ spec: apiVersion: airshipit.org/v1alpha1 kind: ReplacementTransformer metadata: - name: notImportantHere + name: Test_Case_13 replacements: - source: objref: @@ -742,7 +811,7 @@ spec: apiVersion: airshipit.org/v1alpha1 kind: ReplacementTransformer metadata: - name: notImportantHere + name: Test_Case_14 replacements: - source: objref: @@ -781,7 +850,7 @@ spec: apiVersion: airshipit.org/v1alpha1 kind: ReplacementTransformer metadata: - name: notImportantHere + name: Test_Case_15 replacements: - source: objref: @@ -822,7 +891,7 @@ spec: apiVersion: airshipit.org/v1alpha1 kind: ReplacementTransformer metadata: - name: notImportantHere + name: Test_Case_16 replacements: - source: objref: @@ -861,7 +930,7 @@ spec: apiVersion: airshipit.org/v1alpha1 kind: ReplacementTransformer metadata: - name: notImportantHere + name: Test_Case_17 replacements: - source: objref: @@ -900,7 +969,7 @@ spec: apiVersion: airshipit.org/v1alpha1 kind: ReplacementTransformer metadata: - name: notImportantHere + name: Test_Case_18 replacements: - source: objref: @@ -932,53 +1001,14 @@ spec: containers: - image: nginx:TAG name: nginx-latest`, - expectedErr: "pattern-based substitution can only be applied to string target fields", + expectedErr: "pattern-based substitution can only be applied to string or array of strings target fields", }, { cfg: ` apiVersion: airshipit.org/v1alpha1 kind: ReplacementTransformer metadata: - name: notImportantHere -replacements: -- source: - objref: - kind: Pod - name: pod1 - target: - objref: - kind: Deployment - fieldrefs: - - spec.template.spec.containers[name=nginx-latest].image%TAG%`, - in: ` -apiVersion: v1 -kind: Pod -metadata: - name: pod1 -spec: - containers: - - name: myapp-container - image: busybox ---- -group: apps -apiVersion: v1 -kind: Deployment -metadata: - name: deploy1 -spec: - template: - spec: - containers: - - image: nginx:latest - name: nginx-latest`, - expectedErr: "pattern 'TAG' is defined in configuration but was not found in target value nginx:latest", - }, - { - cfg: ` -apiVersion: airshipit.org/v1alpha1 -kind: ReplacementTransformer -metadata: - name: notImportantHere + name: Test_Case_19 replacements: - source: value: "12345678" @@ -1050,15 +1080,15 @@ func TestReplacementTransformer(t *testing.T) { func TestExec(t *testing.T) { // TODO (dukov) Remove this once we migrate to new kustomize plugin approach // NOTE (dukov) we need this since error format is different for new kustomize plugins - testCases[11].expectedErr = "wrong Node Kind for labels.somelabel expected: " + + testCases[13].expectedErr = "wrong Node Kind for labels.somelabel expected: " + "MappingNode was ScalarNode: value: {'some string value'}" - testCases[12].expectedErr = "wrong Node Kind for labels.somelabel expected: " + + testCases[14].expectedErr = "wrong Node Kind for labels.somelabel expected: " + "SequenceNode was ScalarNode: value: {'some string value'}" - testCases[13].expectedErr = "wrong Node Kind for spec expected: " + + testCases[15].expectedErr = "wrong Node Kind for spec expected: " + "SequenceNode was MappingNode: value: {containers:\n- name: myapp-container\n image: busybox}" - testCases[15].expectedErr = "wrong Node Kind for spec.containers expected: " + + testCases[17].expectedErr = "wrong Node Kind for spec.containers expected: " + "MappingNode was SequenceNode: value: {- name: myapp-container\n image: busybox}" - testCases[16].expectedErr = "wrong Node Kind for expected: " + + testCases[18].expectedErr = "wrong Node Kind for expected: " + "ScalarNode was MappingNode: value: {image: nginx:TAG\nname: nginx-latest}" for i, tc := range testCases {