From 025c2172d61f3a7f24b17343b7a686b20a2af70d Mon Sep 17 00:00:00 2001 From: Dmitry Ukov Date: Wed, 23 Sep 2020 18:01:17 +0400 Subject: [PATCH] Implement JSON path filter for kyaml.RNode Change introduces kyaml.RNode Pipe Filter which uses k8s go-client JSON path parser. This enables to use JSON queries defined by https://goessner.net/articles/JsonPath/ Change-Id: I6c2276f27652190ed9d183cea0e45eb118346c6b Relates-To: #340 --- pkg/document/plugin/kyamlutils/errors.go | 69 +++ .../plugin/kyamlutils/jsonpathfilter.go | 528 ++++++++++++++++++ .../plugin/kyamlutils/jsonpathfilter_test.go | 477 ++++++++++++++++ pkg/document/plugin/replacement/errors.go | 5 +- .../plugin/replacement/transformer.go | 2 +- .../plugin/replacement/transformer_test.go | 2 +- 6 files changed, 1079 insertions(+), 4 deletions(-) create mode 100644 pkg/document/plugin/kyamlutils/errors.go create mode 100644 pkg/document/plugin/kyamlutils/jsonpathfilter.go create mode 100644 pkg/document/plugin/kyamlutils/jsonpathfilter_test.go diff --git a/pkg/document/plugin/kyamlutils/errors.go b/pkg/document/plugin/kyamlutils/errors.go new file mode 100644 index 000000000..6bc1db268 --- /dev/null +++ b/pkg/document/plugin/kyamlutils/errors.go @@ -0,0 +1,69 @@ +/* + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package kyamlutils + +import ( + "fmt" + + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +// ErrIndexOutOfBound returned if JSON path points to a wrong index of a list +type ErrIndexOutOfBound struct { + Index int + Length int +} + +func (e ErrIndexOutOfBound) Error() string { + return fmt.Sprintf("array index out of bounds: index %d, length %d", e.Index, e.Length) +} + +// ErrBadQueryFormat raised for JSON query errors +type ErrBadQueryFormat struct { + Msg string +} + +func (e ErrBadQueryFormat) Error() string { + return e.Msg +} + +// ErrLookup raised if error occurs during walk through yaml resource +type ErrLookup struct { + Msg string +} + +func (e ErrLookup) Error() string { + return e.Msg +} + +// ErrNotScalar returned if value defined by key in JSON path is not scalar +// Error can be returned for filter queries +type ErrNotScalar struct { + Node *yaml.Node +} + +func (e ErrNotScalar) Error() string { + return fmt.Sprintf("%#v is not scalar", e.Node) +} + +// ErrQueryConversion returned by query conversion function +type ErrQueryConversion struct { + Msg string + Query string +} + +func (e ErrQueryConversion) Error() string { + return fmt.Sprintf("failed to convert v1 path '%s' to jsonpath. %s", e.Query, e.Msg) +} diff --git a/pkg/document/plugin/kyamlutils/jsonpathfilter.go b/pkg/document/plugin/kyamlutils/jsonpathfilter.go new file mode 100644 index 000000000..cebf3417b --- /dev/null +++ b/pkg/document/plugin/kyamlutils/jsonpathfilter.go @@ -0,0 +1,528 @@ +/* + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package kyamlutils + +import ( + "fmt" + "strconv" + "strings" + + "k8s.io/client-go/util/jsonpath" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +// JSONPathFilter returns RNode under JSON path +type JSONPathFilter struct { + Kind string `yaml:"kind,omitempty"` + // Path is jsonpath query. See http://goessner.net/articles/JsonPath/ for + // details. + Path string `yaml:"path,omitempty"` + // Mutator is a function which changes filtered node + Mutator func([]*yaml.RNode) error + // Create empty struct if path element does not exist + Create bool +} + +type boundaries struct { + min, max, step int +} + +// Filter returns RNode identified by JSON path +func (l JSONPathFilter) Filter(rn *yaml.RNode) (*yaml.RNode, error) { + query, err := convertFromLegacyQuery(l.Path) + if err != nil { + return nil, err + } + jp, err := jsonpath.Parse("parser", query) + if err != nil { + return nil, err + } + if len(jp.Root.Nodes) != 1 { + return nil, ErrBadQueryFormat{Msg: "query must contain one expression"} + } + + fieldList, ok := jp.Root.Nodes[0].(*jsonpath.ListNode) + if !ok { + return nil, ErrBadQueryFormat{Msg: "failed to convert root node to ListNode type"} + } + // Query result can be a list of values (e.g filter queries) since + // walk is recursive we should pass list of RNode even for initial step + rns, err := l.walk([]*yaml.RNode{rn}, fieldList) + if len(rns) == 0 || err != nil { + return nil, err + } + + if l.Mutator != nil { + if err := l.Mutator(rns); err != nil { + return nil, err + } + } + + // Return first element if filtered list contins only one RNode + // otherwise create a SequenceNode + if len(rns) == 1 { + return rns[0], nil + } + nodes := make([]*yaml.Node, len(rns)) + for i := range rns { + nodes[i] = rns[i].YNode() + } + return yaml.NewRNode(&yaml.Node{Kind: yaml.SequenceNode, Content: nodes}), nil +} + +func (l JSONPathFilter) walk(nodes []*yaml.RNode, fieldList *jsonpath.ListNode) (res []*yaml.RNode, err error) { + res = nodes + for _, field := range fieldList.Nodes { + res, err = l.getByField(res, field) + if err != nil { + return nil, err + } + } + return res, nil +} + +func (l JSONPathFilter) getByField(rns []*yaml.RNode, field jsonpath.Node) ([]*yaml.RNode, error) { + switch fieldT := field.(type) { + // Can be encountered in filter expressions e.g. [?(10 == 10)]; 10 is IntNode + case *jsonpath.IntNode: + return []*yaml.RNode{ + yaml.NewRNode(&yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!int", + Value: strconv.Itoa(fieldT.Value), + }), + }, nil + // Same as IntNode + case *jsonpath.FloatNode: + return []*yaml.RNode{ + yaml.NewRNode(&yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!float", + Value: strconv.FormatFloat(fieldT.Value, 'f', -1, 64), + }), + }, nil + // Same as IntNode + case *jsonpath.BoolNode: + return []*yaml.RNode{yaml.NewScalarRNode(strconv.FormatBool(fieldT.Value))}, nil + // Same as IntNode + case *jsonpath.TextNode: + return []*yaml.RNode{yaml.NewScalarRNode(fieldT.Text)}, nil + // For example spec.containers.metadata 'spec', 'containers' and + // 'metadata' have FieldNode type + case *jsonpath.FieldNode: + res, err := l.doField(rns, fieldT) + // Create field node if needed + if len(res) == 0 && l.Create { + res = []*yaml.RNode{yaml.NewMapRNode(nil)} + return res, l.updateMapRNodes(rns, fieldT.Value, res[0]) + } + return res, err + // Field is accessing to array via direct element number or by getting + // a slice e.g. [20] or [2:10:3] + case *jsonpath.ArrayNode: + return l.doArray(rns, fieldT) + // Filter some elements of the array e.g. [?(.name == 'someName')] will + // return all array elements which contain name field equal to 'someName' + case *jsonpath.FilterNode: + return l.doFilter(rns, fieldT) + // Return all elements e.g. spec.containers[*] + case *jsonpath.WildcardNode: + return l.doWildcard(rns) + } + return nil, ErrBadQueryFormat{Msg: fmt.Sprintf("unsupported jsonpath expression %s", l.Path)} +} + +func (l JSONPathFilter) doField(rns []*yaml.RNode, field *jsonpath.FieldNode) ([]*yaml.RNode, error) { + // NOTE this code block was adapted from k8s.io/client-go/util/jsonpath + // see evalField from jsonpath.go + result := []*yaml.RNode{} + if len(rns) == 0 { + return result, nil + } + if field.Value == "" { + return rns, nil + } + for _, rn := range rns { + res, err := rn.Pipe(yaml.Get(field.Value)) + if err != nil { + return nil, err + } + if res != nil { + res.AppendToFieldPath(append(rn.FieldPath(), field.Value)...) + result = append(result, res) + } + } + return result, nil +} + +func (l JSONPathFilter) doArray(rns []*yaml.RNode, field *jsonpath.ArrayNode) ([]*yaml.RNode, error) { + // NOTE this code block was adapted from k8s.io/client-go/util/jsonpath + // see evalArray from jsonpath.go + result := []*yaml.RNode{} + for _, rn := range rns { + if err := yaml.ErrorIfInvalid(rn, yaml.SequenceNode); err != nil { + return nil, err + } + bounds, err := l.defineArrayBoundaries(field.Params, len(rn.Content())) + if err != nil { + return nil, err + } + + if bounds.min == bounds.max { + return result, nil + } + + for i := bounds.min; i < bounds.max; i += bounds.step { + result = append(result, yaml.NewRNode(rn.Content()[i])) + } + } + return result, nil +} + +func (l JSONPathFilter) defineArrayBoundaries(params [3]jsonpath.ParamsEntry, maxLen int) (boundaries, error) { + res := boundaries{ + min: 0, + max: maxLen, + step: 1, + } + if params[0].Known { + res.min = params[0].Value + } + if params[0].Value < 0 { + res.min = params[0].Value + maxLen + } + if params[1].Known { + res.max = params[1].Value + } + + if params[1].Value < 0 || (params[1].Value == 0 && params[1].Derived) { + res.max = params[1].Value + maxLen + } + + if res.min >= maxLen || res.min < 0 { + return boundaries{min: -1, max: -1, step: -1}, ErrIndexOutOfBound{Index: params[0].Value, Length: maxLen} + } + if res.max > maxLen || res.max < 0 { + return boundaries{min: -1, max: -1, step: -1}, ErrIndexOutOfBound{Index: params[1].Value - 1, Length: maxLen} + } + if res.min > res.max { + return boundaries{min: -1, max: -1, step: -1}, ErrLookup{ + fmt.Sprintf("starting index %d is greater than ending index %d", + params[0].Value, params[1].Value), + } + } + + if params[2].Known { + if params[2].Value <= 0 { + return boundaries{min: -1, max: -1, step: -1}, ErrLookup{fmt.Sprintf("step must be > 0")} + } + res.step = params[2].Value + } + + return res, nil +} + +func (l JSONPathFilter) doFilter(rns []*yaml.RNode, field *jsonpath.FilterNode) ([]*yaml.RNode, error) { + // NOTE this cond block was adapted from k8s.io/client-go/util/jsonpath + // see evalFilter from from jsonpath.go + result := []*yaml.RNode{} + for _, rn := range rns { + // Elements() will return an error if rn is not SequenceNode + // therefore no need to check it explicitly + items, err := rn.Elements() + if err != nil { + return nil, err + } + var left, right []*yaml.RNode + for _, item := range items { + pass := false + // Evaluate left part of the filter expression against each + // array item + left, err = l.walk([]*yaml.RNode{item}, field.Left) + if err != nil { + return nil, err + } + + if len(left) == 0 { + continue + } + if len(left) > 1 { + return nil, ErrBadQueryFormat{Msg: "can only compare one element at a time"} + } + + // Evaluate right part of the filter expression against each + // array item + right, err = l.walk([]*yaml.RNode{item}, field.Right) + if err != nil { + return nil, err + } + + if len(right) == 0 { + continue + } + if len(right) > 1 { + return nil, ErrBadQueryFormat{Msg: "can only compare one element at a time"} + } + + pass, err = l.compare(field.Operator, left[0], right[0]) + if err != nil { + return nil, err + } + if pass { + result = append(result, item) + } + } + } + return result, nil +} + +func (l JSONPathFilter) compare(op string, left, right *yaml.RNode) (bool, error) { + err := yaml.ErrorIfAnyInvalidAndNonNull(yaml.ScalarNode, left, right) + if op != "==" && err != nil { + return false, err + } + switch op { + case "<": + return lt(left, right) + case ">": + return gt(left, right) + case "==": + return eq(left, right) + case "!=": + pass, err := eq(left, right) + if err != nil { + return false, err + } + return !pass, nil + case "<=": + return le(left, right) + case ">=": + return ge(left, right) + } + return false, ErrLookup{Msg: fmt.Sprintf("unrecognized filter operator %s", op)} +} + +func (l JSONPathFilter) doWildcard(rns []*yaml.RNode) ([]*yaml.RNode, error) { + // NOTE this code block was adapted from k8s.io/client-go/util/jsonpath + // see evalArray from jsonpath.go + result := []*yaml.RNode{} + for _, rn := range rns { + switch rn.YNode().Kind { + case yaml.SequenceNode: + items, err := rn.Elements() + if err != nil { + return nil, err + } + result = append(result, items...) + case yaml.MappingNode: + keys, err := rn.Fields() + if err != nil { + return nil, err + } + for _, key := range keys { + result = append(result, rn.Field(key).Value) + } + } + } + return result, nil +} + +func (l JSONPathFilter) updateMapRNodes(rns []*yaml.RNode, name string, node *yaml.RNode) error { + for _, rn := range rns { + _, err := rn.Pipe(yaml.SetField(name, node)) + if err != nil { + return err + } + } + return nil +} + +func eq(a, b *yaml.RNode) (bool, error) { + return deepEq(a.YNode(), b.YNode()) +} + +func lt(a, b *yaml.RNode) (bool, error) { + if a.YNode().Tag == "!!float" || b.YNode().Tag == "!!float" { + af, err := strconv.ParseFloat(a.YNode().Value, 64) + if err != nil { + return false, err + } + bf, err := strconv.ParseFloat(b.YNode().Value, 64) + if err != nil { + return false, err + } + return af < bf, nil + } + + if a.YNode().Tag == "!!int" || b.YNode().Tag == "!!int" { + ai, err := strconv.Atoi(a.YNode().Value) + if err != nil { + return false, err + } + bi, err := strconv.Atoi(b.YNode().Value) + if err != nil { + return false, err + } + return ai < bi, nil + } + return a.YNode().Value < b.YNode().Value, nil +} + +func le(a, b *yaml.RNode) (bool, error) { + if less, err := lt(a, b); less || err != nil { + return less, err + } + return eq(a, b) +} + +func gt(a, b *yaml.RNode) (bool, error) { + return lt(b, a) +} + +func ge(a, b *yaml.RNode) (bool, error) { + return le(b, a) +} + +func scalarEq(a, b *yaml.Node) (bool, error) { + if a.Tag == "!!float" || b.Tag == "!!float" { + af, err := strconv.ParseFloat(a.Value, 64) + if err != nil { + return false, err + } + bf, err := strconv.ParseFloat(b.Value, 64) + if err != nil { + return false, err + } + return af == bf, nil + } + + if a.Tag == "!!int" || b.Tag == "!!int" { + ai, err := strconv.Atoi(a.Value) + if err != nil { + return false, err + } + bi, err := strconv.Atoi(b.Value) + if err != nil { + return false, err + } + return ai == bi, nil + } + return a.Value == b.Value, nil +} + +func deepEq(a, b *yaml.Node) (bool, error) { + if a == nil || b == nil { + return a == b, nil + } + if a.Kind == yaml.ScalarNode || b.Kind == yaml.ScalarNode { + return scalarEq(a, b) + } + if len(a.Content) != len(b.Content) { + return false, nil + } + for i, item := range a.Content { + pass, err := deepEq(item, b.Content[i]) + if !pass || err != nil { + return false, err + } + } + return true, nil +} + +func convertFromLegacyQuery(query string) (string, error) { + path := strings.TrimSpace(query) + if strings.HasPrefix(query, "{") { + return path, nil + } + var result []string + // NOTE this code is borrowed from k8sdeps module of kustomize api + start := 0 + insideParentheses := false + for i, c := range path { + switch c { + case '.': + if !insideParentheses { + result = convertField(path[start:i], result) + start = i + 1 + } + case '[': + if !insideParentheses { + if path[start:i] != "" { + result = append(result, path[start:i]) + } + start = i + 1 + insideParentheses = true + } else { + return "", ErrQueryConversion{Msg: "nested parentheses are not allowed", Query: path} + } + case ']': + if insideParentheses { + // Assign this index to the current + // PathSection, save it to the result, then begin + // a new PathSection + result = convertFiltering(path[start:i], result) + + start = i + 1 + insideParentheses = false + } else { + return "", ErrQueryConversion{Msg: "invalid field path", Query: path} + } + } + } + if start < len(path) { + result = convertField(path[start:], result) + } + return "{." + strings.Join(result, ".") + "}", nil +} + +func convertFiltering(query string, path []string) []string { + _, err := strconv.Atoi(query) + result := path + switch { + case err == nil: + // We have detected an integer so an array. + result[len(result)-1] = result[len(result)-1] + "[" + query + "]" + case strings.Contains(query, "="): + kvFilterParts := strings.Split(query, "=") + q := fmt.Sprintf("[?(.%s == '%s')]", + strings.ReplaceAll(kvFilterParts[0], ".", "\\."), kvFilterParts[1]) + result[len(result)-1] += q + default: + // e.g. spec['containers'] + key := strings.Trim(query, "'") + key = strings.Trim(key, "\"") + key = strings.ReplaceAll(key, ".", "\\.") + result = append(result, key) + } + return result +} + +func convertField(query string, path []string) []string { + if query == "" { + return path + } + _, err := strconv.Atoi(query) + if err != nil { + return append(path, query) + } + + if len(path) == 0 { + return []string{"[" + query + "]"} + } + + result := append([]string(nil), path...) + result[len(result)-1] += "[" + query + "]" + return result +} diff --git a/pkg/document/plugin/kyamlutils/jsonpathfilter_test.go b/pkg/document/plugin/kyamlutils/jsonpathfilter_test.go new file mode 100644 index 000000000..821bfe5b2 --- /dev/null +++ b/pkg/document/plugin/kyamlutils/jsonpathfilter_test.go @@ -0,0 +1,477 @@ +/* + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package kyamlutils_test + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sigs.k8s.io/kustomize/kyaml/yaml" + + "opendev.org/airship/airshipctl/pkg/document/plugin/kyamlutils" +) + +func TestGetterFilter(t *testing.T) { + obj, err := yaml.Parse(`apiVersion: apps/v1 +kind: SomeKind +metadata: + name: resource-name +spec: + path: + to: + some: + scalar: + value: 300 + nonScalar: + key: val + array: + - i1 + - i2 +keyWith.dot: + - subkeyWith.dot: val1 + name: obj1 + - subkeyWith.dot: val2 + name: obj2 +listOfStrings: + - string1 + - string2 + - string3 +listOfObjects: + - name: obj1 + value: 10 + - name: obj2 + value: 20 + - name: obj3 + value: 30 +listOfComplexObjects: + - name: o1 + value: + someKey1: valO11 + someKey2: valO12 + value1: + someKey1: valO11 + someKey2: valO12 + - name: o2 + value: + someKey0: false + someKey1: valO21 + someKey2: valO22 +sameObjects: + - name: otherName + val: otherValue + - name: o1 + val: val1 + - name: o1 + val: val1`) + require.NoError(t, err) + + testCases := []struct { + name string + query string + expectedErr string + expected string + }{ + // Parser cases + { + name: "Multiple expressions", + query: "{.a.b} {.x.y}", + expectedErr: "query must contain one expression", + }, + // Field cases + { + name: "Path to scalar value", + query: "{.spec.path.to.some.scalar.value}", + expected: "300", + }, + { + name: "Key name with dot", + query: "{.keyWith\\.dot}", + expected: ` +- subkeyWith.dot: val1 + name: obj1 +- subkeyWith.dot: val2 + name: obj2`[1:], + }, + { + name: "Field from array of scalar values", + query: "{.listOfStrings.value}", + expectedErr: "wrong Node Kind for listOfStrings " + + "expected: MappingNode was SequenceNode: value: {- string1\n- string2\n- string3}", + }, + { + name: "Non Existent path", + query: "{.a.b}", + }, + // Array cases + { + name: "Array element negative", + query: "{.listOfObjects[-1]}", + expected: ` +name: obj3 +value: 30`[1:], + }, + { + name: "Array element", + query: "{.listOfObjects[0]}", + expected: ` +name: obj1 +value: 10`[1:], + }, + { + name: "Array element value", + query: "{.listOfObjects[0].value}", + expected: "10", + }, + { + name: "From element to end", + query: "{.listOfObjects[1:]}", + expected: ` +- name: obj2 + value: 20 +- name: obj3 + value: 30`[1:], + }, + { + name: "Segment", + query: "{.listOfObjects[1:3]}", + expected: ` +- name: obj2 + value: 20 +- name: obj3 + value: 30`[1:], + }, + { + name: "Segment with negative right", + query: "{.listOfObjects[1:-1]}", + expected: ` +name: obj2 +value: 20`[1:], + }, + { + name: "From start to element", + query: "{.listOfObjects[:2]}", + expected: ` +- name: obj1 + value: 10 +- name: obj2 + value: 20`[1:], + }, + { + name: "Iterate with step", + query: "{.listOfObjects[::2]}", + expected: ` +- name: obj1 + value: 10 +- name: obj3 + value: 30`[1:], + }, + { + name: "Empty list", + query: "{.listOfObjects[1:1]}", + expected: "", + }, + { + name: "Access to non array", + query: "{.metadata[1]}", + expectedErr: "wrong Node Kind for metadata expected: SequenceNode was MappingNode: value: {name: resource-name}", + }, + { + name: "Out of left bound", + query: "{.listOfObjects[100]}", + expectedErr: "array index out of bounds: index 100, length 3", + }, + { + name: "Out of right bound", + query: "{.listOfObjects[:100]}", + expectedErr: "array index out of bounds: index 99, length 3", + }, + { + name: "Bad bounds", + query: "{.listOfObjects[2:1]}", + expectedErr: "starting index 2 is greater than ending index 1", + }, + { + name: "Negative step", + query: "{.listOfObjects[::-2]}", + expectedErr: "step must be > 0", + }, + // Filter cases + { + name: "Filter by key", + query: "{.listOfObjects[?(.name == 'obj1')]}", + expected: ` +name: obj1 +value: 10`[1:], + }, + { + name: "Filter by key name with dot", + query: "{.keyWith\\.dot[?(.subkeyWith\\.dot == 'val2')]}", + expected: ` +subkeyWith.dot: val2 +name: obj2`[1:], + }, + { + name: "Filter eq int", + query: "{.listOfObjects[?(.value == 10)]}", + expected: ` +name: obj1 +value: 10`[1:], + }, + { + name: "Filter by content", + query: "{.listOfStrings[?(. == 'string1')]}", + expected: "string1", + }, + { + name: "Query parse error", + query: "{.listOfStrings[?(. == 'string1'", + expectedErr: "unterminated filter", + }, + { + name: "Filter non-array", + query: "{.metadata[?(.name == 'name')]}", + expectedErr: "wrong Node Kind for metadata expected: SequenceNode was MappingNode: value: {name: resource-name}", + }, + { + name: "Filter condition non-array", + query: "{.listOfObjects[?(.[0] == 'name')]}", + expectedErr: "wrong Node Kind for expected: SequenceNode was MappingNode: value: {name: obj1\nvalue: 10}", + }, + { + name: "Filter eq non-scalars", + query: "{.listOfComplexObjects[?(.value == 'name')]}", + expected: "", + }, + { + name: "Filter compare non-scalars", + query: "{.listOfComplexObjects[?(.value > 'name')]}", + expectedErr: "wrong Node Kind for value expected: ScalarNode was MappingNode: value: {someKey1: valO11\n" + + "someKey2: valO12}", + }, + { + name: "Filter unknown operator", + query: "{.listOfStrings[?(. >> 'name')]}", + expectedErr: "unrecognized filter operator >>", + }, + { + name: "Filter gt int", + query: "{.listOfObjects[?(.value > 10)]}", + expected: ` +- name: obj2 + value: 20 +- name: obj3 + value: 30`[1:], + }, + { + name: "Filter gt float", + query: "{.listOfObjects[?(.value > 7.5)]}", + expected: ` +- name: obj1 + value: 10 +- name: obj2 + value: 20 +- name: obj3 + value: 30`[1:], + }, + { + name: "Filter ge float", + query: "{.listOfObjects[?(.value >= 20.0)]}", + expected: ` +- name: obj2 + value: 20 +- name: obj3 + value: 30`[1:], + }, + { + name: "Filter lt int", + query: "{.listOfObjects[?(20 < .value)]}", + expected: ` +name: obj3 +value: 30`[1:], + }, + { + name: "Filter eq objects", + query: "{.listOfObjects[?(@ == @)]}", + expected: ` +- name: obj1 + value: 10 +- name: obj2 + value: 20 +- name: obj3 + value: 30`[1:], + }, + { + name: "Filter eq sub-objects", + query: "{.listOfComplexObjects[?(.value == .value1)]}", + expected: ` +name: o1 +value: + someKey1: valO11 + someKey2: valO12 +value1: + someKey1: valO11 + someKey2: valO12`[1:], + }, + { + name: "Filter not eq error", + query: "{.listOfObjects[?(.name != 10)]}", + expectedErr: `strconv.Atoi: parsing "obj1": invalid syntax`, + }, + { + name: "Filter eq bool", + query: "{.listOfComplexObjects[?(.value.someKey0 == false)]}", + expected: ` +name: o2 +value: + someKey0: false + someKey1: valO21 + someKey2: valO22`[1:], + }, + // Wildcard cases + { + name: "Get map by wildcard", + query: "{.spec.path.to.some.*}", + expected: ` +- value: 300 +- key: val +- - i1 + - i2`[1:], + }, + { + name: "Wildcard list", + query: "{.listOfStrings.*}", + expected: ` +- string1 +- string2 +- string3`[1:], + }, + // v1 queries cases + { + name: "Array element v1", + query: "listOfObjects[0]", + expected: ` +name: obj1 +value: 10`[1:], + }, + { + name: "Array element value v1", + query: "listOfObjects[0].value", + expected: "10", + }, + { + name: "Array element by numeric key v1", + query: "listOfObjects.0", + expected: ` +name: obj1 +value: 10`[1:], + }, + { + name: "Array element value by numeric key v1", + query: "listOfObjects.0.value", + expected: "10", + }, + { + name: "Array filter v1", + query: "listOfObjects[name=obj1]", + expected: ` +name: obj1 +value: 10`[1:], + }, + { + name: "Array filter get all value v1", + query: "listOfObjects[name=obj1].value", + expected: "10", + }, + { + name: "Array filter get same value v1", + query: "sameObjects[name=o1].val", + expected: ` +- val1 +- val1`[1:], + }, + { + name: "Array filter get first value v1", + query: "['keyWith.dot'][subkeyWith.dot=val1]", + expected: ` +subkeyWith.dot: val1 +name: obj1`[1:], + }, + { + name: "Nested parentheses", + query: "spec[path[0]]", + expectedErr: "failed to convert v1 path 'spec[path[0]]' to jsonpath. nested parentheses are not allowed", + }, + { + name: "Invalid closing parentheses", + query: "spec[path]]", + expectedErr: "failed to convert v1 path 'spec[path]]' to jsonpath. invalid field path", + }, + } + for _, tc := range testCases { + tt := tc + t.Run(tt.name, func(t *testing.T) { + getter := kyamlutils.JSONPathFilter{Path: tt.query} + actualObj, err := obj.Pipe(getter) + errS := "" + if err != nil { + errS = err.Error() + } + assert.Equal(t, tc.expectedErr, errS) + actualString, err := actualObj.String() + require.NoError(t, err) + assert.Equal(t, tc.expected, strings.TrimSuffix(actualString, "\n")) + }) + } +} + +func TestFilterCreate(t *testing.T) { + data := `apiVersion: apps/v1 +kind: SomeKind +metadata: + name: resource-name` + testCases := []struct { + name string + query string + create bool + }{ + { + name: "Create if not exists", + query: "{.spec.not.exists}", + create: true, + }, + { + name: "Not exists", + query: "{.spec.not.exists}", + }, + } + + for _, tc := range testCases { + tt := tc + t.Run(tt.name, func(t *testing.T) { + obj, err := yaml.Parse(data) + require.NoError(t, err) + getter := kyamlutils.JSONPathFilter{Path: tt.query, Create: tt.create} + actualObj, err := obj.Pipe(getter) + require.NoError(t, err) + if tt.create { + assert.NotNil(t, actualObj) + } else { + assert.Nil(t, actualObj) + } + }) + } +} diff --git a/pkg/document/plugin/replacement/errors.go b/pkg/document/plugin/replacement/errors.go index a73ada0e1..5fd99428b 100644 --- a/pkg/document/plugin/replacement/errors.go +++ b/pkg/document/plugin/replacement/errors.go @@ -106,11 +106,12 @@ func (e ErrPatternSubstring) Error() string { // ErrIndexOutOfBound returned if JSON path points to a wrong index of a list type ErrIndexOutOfBound struct { - Index int + Index int + Length int } func (e ErrIndexOutOfBound) Error() string { - return fmt.Sprintf("index %v is out of bound", e.Index) + return fmt.Sprintf("array index out of bounds: index %d, length %d", e.Index, e.Length) } // ErrMapNotFound returned if map specified in fieldRef option was not found in a list diff --git a/pkg/document/plugin/replacement/transformer.go b/pkg/document/plugin/replacement/transformer.go index aa7b4f2d2..4d89a39c8 100644 --- a/pkg/document/plugin/replacement/transformer.go +++ b/pkg/document/plugin/replacement/transformer.go @@ -328,7 +328,7 @@ func updateSliceField(m []interface{}, pathToField []string, replacement interfa } if len(m) < index || index < 0 { - return ErrIndexOutOfBound{Index: index} + return ErrIndexOutOfBound{Index: index, Length: len(m)} } if len(pathToField) == 1 { m[index] = replacement diff --git a/pkg/document/plugin/replacement/transformer_test.go b/pkg/document/plugin/replacement/transformer_test.go index 7047fb8f8..92184a117 100644 --- a/pkg/document/plugin/replacement/transformer_test.go +++ b/pkg/document/plugin/replacement/transformer_test.go @@ -852,7 +852,7 @@ spec: containers: - name: myapp-container image: busybox`, - expectedErr: "index 10 is out of bound", + expectedErr: "array index out of bounds: index 10, length 1", }, { cfg: `