From ef76437ac0f90ab57e99599fc7c3fe7358cc09fc Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 18 Sep 2025 16:41:16 +0200 Subject: [PATCH 01/20] internal: structpath: accurate struct <-> string mapping Previously structpath could represent AnyKey and AnyIndex as struct but as string they were both represented as field[*]. On the other hand, there was no unambigous representation of x.*. This commit fixes struct mapping to match string representation 1-1 so that Parse(x.String()) == x. --- bundle/direct/dresources/all_test.go | 8 +- bundle/internal/validation/enum.go | 3 +- bundle/internal/validation/required.go | 3 +- libs/structs/dynpath/dynpath.go | 101 ++++++++++++ libs/structs/dynpath/dynpath_test.go | 202 +++++++++++++++++++++++ libs/structs/structaccess/typecheck.go | 22 ++- libs/structs/structdiff/diff.go | 11 +- libs/structs/structpath/path.go | 126 +++++++------- libs/structs/structpath/path_test.go | 188 +++++++++------------ libs/structs/structwalk/walk.go | 8 +- libs/structs/structwalk/walktype.go | 11 +- libs/structs/structwalk/walktype_test.go | 10 +- 12 files changed, 501 insertions(+), 192 deletions(-) create mode 100644 libs/structs/dynpath/dynpath.go create mode 100644 libs/structs/dynpath/dynpath_test.go diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 18af7bf4e9..7fee2829b7 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/dynpath" "github.com/databricks/cli/libs/structs/structaccess" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/structs/structwalk" @@ -155,9 +156,10 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W require.NotNil(t, remappedState) require.NoError(t, structwalk.Walk(newState, func(path *structpath.PathNode, val any, field *reflect.StructField) { - remoteValue, err := structaccess.Get(remappedState, dyn.MustPathFromString(path.DynPath())) + dynPath := dynpath.ConvertPathNodeToDynPath(path, reflect.TypeOf(newState)) + remoteValue, err := structaccess.Get(remappedState, dyn.MustPathFromString(dynPath)) if err != nil { - t.Errorf("Failed to read %s from remapped remote state %#v", path.DynPath(), remappedState) + t.Errorf("Failed to read %s from remapped remote state %#v", dynPath, remappedState) } if val == nil { // t.Logf("Ignoring %s nil, remoteValue=%#v", path.String(), remoteValue) @@ -172,7 +174,7 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W // t.Logf("Testing %s v=%#v, remoteValue=%#v", path.String(), val, remoteValue) // We expect fields set explicitly to be preserved by testserver, which is true for all resources as of today. // If not true for your resource, add exception here: - assert.Equal(t, val, remoteValue, path.DynPath()) + assert.Equal(t, val, remoteValue, dynPath) })) err = adapter.DoDelete(ctx, createdID) diff --git a/bundle/internal/validation/enum.go b/bundle/internal/validation/enum.go index cd5bfc2044..d48cfa9dc2 100644 --- a/bundle/internal/validation/enum.go +++ b/bundle/internal/validation/enum.go @@ -11,6 +11,7 @@ import ( "text/template" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/libs/structs/dynpath" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/structs/structtag" "github.com/databricks/cli/libs/structs/structwalk" @@ -132,7 +133,7 @@ func extractEnumFields(typ reflect.Type) ([]EnumPatternInfo, error) { return true } - fieldPath := path.DynPath() + fieldPath := dynpath.ConvertPathNodeToDynPath(path, typ) fieldsByPattern[fieldPath] = enumValues } return true diff --git a/bundle/internal/validation/required.go b/bundle/internal/validation/required.go index 72e8f15fab..8de2f0e589 100644 --- a/bundle/internal/validation/required.go +++ b/bundle/internal/validation/required.go @@ -11,6 +11,7 @@ import ( "text/template" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/libs/structs/dynpath" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/structs/structtag" "github.com/databricks/cli/libs/structs/structwalk" @@ -68,7 +69,7 @@ func extractRequiredFields(typ reflect.Type) ([]RequiredPatternInfo, error) { return true } - parentPath := path.Parent().DynPath() + parentPath := dynpath.ConvertPathNodeToDynPath(path.Parent(), typ) fieldsByPattern[parentPath] = append(fieldsByPattern[parentPath], fieldName) return true }) diff --git a/libs/structs/dynpath/dynpath.go b/libs/structs/dynpath/dynpath.go new file mode 100644 index 0000000000..50744919c1 --- /dev/null +++ b/libs/structs/dynpath/dynpath.go @@ -0,0 +1,101 @@ +package dynpath + +import ( + "reflect" + "strconv" + "strings" + + "github.com/databricks/cli/libs/structs/structaccess" + "github.com/databricks/cli/libs/structs/structpath" +) + +// ConvertPathNodeToDynPath converts a PathNode to dyn path format string. +// Uses the provided root type to determine context-aware wildcard formatting: +// - BracketStar accessing maps renders as "parent.*" +// - BracketStar accessing arrays/slices renders as "parent[*]" +// - DotStar always renders as "parent.*" +func ConvertPathNodeToDynPath(path *structpath.PathNode, rootType reflect.Type) string { + if path == nil { + return "" + } + + segments := path.AsSlice() + var result strings.Builder + currentType := rootType + + for _, segment := range segments { + // Dereference pointers + for currentType != nil && currentType.Kind() == reflect.Pointer { + currentType = currentType.Elem() + } + + if index, ok := segment.Index(); ok { + // Array/slice index access + result.WriteString("[") + result.WriteString(strconv.Itoa(index)) + result.WriteString("]") + if currentType != nil && (currentType.Kind() == reflect.Array || currentType.Kind() == reflect.Slice) { + currentType = currentType.Elem() + } else { + currentType = nil + } + + } else if field, ok := segment.Field(); ok { + // Struct field access + if result.Len() > 0 { + result.WriteString(".") + } + result.WriteString(field) + if currentType != nil && currentType.Kind() == reflect.Struct { + if sf, _, ok := structaccess.FindStructFieldByKeyType(currentType, field); ok { + currentType = sf.Type + } else { + currentType = nil + } + } else { + currentType = nil + } + + } else if key, ok := segment.MapKey(); ok { + // Map key access - always use dot notation in DynPath + if result.Len() > 0 { + result.WriteString(".") + } + result.WriteString(key) + if currentType != nil && currentType.Kind() == reflect.Map { + currentType = currentType.Elem() + } else { + currentType = nil + } + + } else if segment.DotStar() { + // Field wildcard - always uses dot notation + if result.Len() > 0 { + result.WriteString(".") + } + result.WriteString("*") + // Type doesn't change for wildcards + + } else if segment.BracketStar() { + // Context-aware wildcard rendering + if currentType != nil && currentType.Kind() == reflect.Map { + // Map wildcard uses dot notation in DynPath + if result.Len() > 0 { + result.WriteString(".") + } + result.WriteString("*") + currentType = currentType.Elem() + } else { + // Array/slice wildcard or fallback uses bracket notation + result.WriteString("[*]") + if currentType != nil && (currentType.Kind() == reflect.Array || currentType.Kind() == reflect.Slice) { + currentType = currentType.Elem() + } else { + currentType = nil + } + } + } + } + + return result.String() +} diff --git a/libs/structs/dynpath/dynpath_test.go b/libs/structs/dynpath/dynpath_test.go new file mode 100644 index 0000000000..0a89e6d966 --- /dev/null +++ b/libs/structs/dynpath/dynpath_test.go @@ -0,0 +1,202 @@ +package dynpath + +import ( + "reflect" + "testing" + + "github.com/databricks/cli/libs/structs/structpath" + "github.com/stretchr/testify/assert" +) + +func TestConvertPathNodeToDynPath(t *testing.T) { + tests := []struct { + name string + structPath string // PathNode string representation to be parsed + dynPath string // Expected DynPath format + rootType reflect.Type // optional, for context-aware wildcard rendering + }{ + // Basic node types + { + name: "nil path", + structPath: "", + dynPath: "", + }, + { + name: "array index", + structPath: "[5]", + dynPath: "[5]", + }, + { + name: "map key", + structPath: "['mykey']", + dynPath: "mykey", + }, + { + name: "struct field with JSON tag", + structPath: "json_name", + dynPath: "json_name", + }, + { + name: "struct field without JSON tag", + structPath: "GoFieldName", + dynPath: "GoFieldName", + }, + { + name: "dot star", + structPath: "*", + dynPath: "*", + }, + { + name: "bracket star - array type", + structPath: "[*]", + rootType: reflect.TypeOf([]int{}), + dynPath: "[*]", + }, + { + name: "bracket star - map type", + structPath: "[*]", + rootType: reflect.TypeOf(map[string]int{}), + dynPath: "*", + }, + + // Compound paths + { + name: "struct field -> array index", + structPath: "items[3]", + dynPath: "items[3]", + }, + { + name: "struct field -> map key", + structPath: "config['database']", + dynPath: "config.database", + }, + { + name: "struct field -> struct field", + structPath: "user.name", + dynPath: "user.name", + }, + { + name: "map key -> array index", + structPath: "['servers'][0]", + dynPath: "servers[0]", + }, + { + name: "map key -> struct field", + structPath: "['primary'].host", + dynPath: "primary.host", + }, + { + name: "array index -> struct field", + structPath: "[2].id", + dynPath: "[2].id", + }, + { + name: "array index -> map key", + structPath: "[1]['status']", + dynPath: "[1].status", + }, + + // Wildcard combinations + { + name: "dot star with parent", + structPath: "Parent.*", + dynPath: "Parent.*", + }, + { + name: "bracket star with parent - array", + structPath: "Parent[*]", + rootType: reflect.TypeOf(struct{ Parent []int }{}), + dynPath: "Parent[*]", + }, + { + name: "bracket star with parent - map", + structPath: "Parent[*]", + rootType: reflect.TypeOf(struct{ Parent map[string]int }{}), + dynPath: "Parent.*", + }, + + // Special characters and edge cases + { + name: "map key with single quote", + structPath: "['key''s']", + dynPath: "key's", + }, + { + name: "map key with multiple single quotes", + structPath: "['''''']", + dynPath: "''", + }, + { + name: "empty map key", + structPath: "['']", + dynPath: "", + }, + { + name: "map key with reserved characters", + structPath: "['key\x00[],`']", + dynPath: "key\x00[],`", + }, + { + name: "field with special characters", + structPath: "field@name:with#symbols!", + dynPath: "field@name:with#symbols!", + }, + { + name: "field with spaces", + structPath: "field with spaces", + dynPath: "field with spaces", + }, + { + name: "field with unicode", + structPath: "名前🙂", + dynPath: "名前🙂", + }, + + // Complex real-world example + { + name: "complex nested path", + structPath: "user.settings['theme'][0].color", + dynPath: "user.settings.theme[0].color", + }, + + // Mixed JSON tag and Go field name scenarios + { + name: "Go field -> JSON field", + structPath: "Parent.child_name", + dynPath: "Parent.child_name", + }, + { + name: "JSON field -> Go field", + structPath: "parent.ChildName", + dynPath: "parent.ChildName", + }, + { + name: "dash JSON tag field", + structPath: "-", + dynPath: "-", + }, + { + name: "JSON tag with options", + structPath: "lazy_field", + dynPath: "lazy_field", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Parse the structPath string into a PathNode + var pathNode *structpath.PathNode + var err error + if tt.structPath == "" { + pathNode = nil + } else { + pathNode, err = structpath.Parse(tt.structPath) + assert.NoError(t, err, "Failed to parse structPath: %s", tt.structPath) + } + + // Convert to DynPath and verify result + result := ConvertPathNodeToDynPath(pathNode, tt.rootType) + assert.Equal(t, tt.dynPath, result, "ConvertPathNodeToDynPath conversion should match expected DynPath format") + }) + } +} diff --git a/libs/structs/structaccess/typecheck.go b/libs/structs/structaccess/typecheck.go index a7b804c748..2582c963cd 100644 --- a/libs/structs/structaccess/typecheck.go +++ b/libs/structs/structaccess/typecheck.go @@ -51,7 +51,7 @@ func Validate(t reflect.Type, path dyn.Path) error { switch cur.Kind() { case reflect.Struct: - sf, _, ok := findStructFieldByKeyType(cur, c.Key()) + sf, _, ok := FindStructFieldByKeyType(cur, c.Key()) if !ok { return fmt.Errorf("%s: field %q not found in %s", newPrefix, c.Key(), cur.String()) } @@ -83,11 +83,15 @@ func Validate(t reflect.Type, path dyn.Path) error { return nil } -// findStructFieldByKeyType searches exported fields of struct type t for a field matching key. +// FindStructFieldByKeyType searches exported fields of struct type t for a field matching key. // It matches json tag name (when present and not "-") only. // It also searches embedded anonymous structs (pointer or value) recursively. // Returns the StructField, the declaring owner type, and whether it was found. -func findStructFieldByKeyType(t reflect.Type, key string) (reflect.StructField, reflect.Type, bool) { +func FindStructFieldByKeyType(t reflect.Type, key string) (reflect.StructField, reflect.Type, bool) { + if t.Kind() != reflect.Struct { + return reflect.StructField{}, reflect.TypeOf(nil), false + } + // First pass: direct fields for i := range t.NumField() { sf := t.Field(i) @@ -106,6 +110,16 @@ func findStructFieldByKeyType(t reflect.Type, key string) (reflect.StructField, } return sf, t, true } + + // Fallback to Go field name when no JSON tag + if name == "" && sf.Name == key { + // Skip fields marked as internal/readonly + btag := structtag.BundleTag(sf.Tag.Get("bundle")) + if btag.Internal() || btag.ReadOnly() { + continue + } + return sf, t, true + } } // Second pass: search embedded anonymous structs recursively (flattening semantics) @@ -121,7 +135,7 @@ func findStructFieldByKeyType(t reflect.Type, key string) (reflect.StructField, if ft.Kind() != reflect.Struct { continue } - if osf, owner, ok := findStructFieldByKeyType(ft, key); ok { + if osf, owner, ok := FindStructFieldByKeyType(ft, key); ok { // Skip fields marked as internal/readonly btag := structtag.BundleTag(osf.Tag.Get("bundle")) if btag.Internal() || btag.ReadOnly() { diff --git a/libs/structs/structdiff/diff.go b/libs/structs/structdiff/diff.go index 473e8b26de..e86e323b84 100644 --- a/libs/structs/structdiff/diff.go +++ b/libs/structs/structdiff/diff.go @@ -127,7 +127,15 @@ func diffStruct(path *structpath.PathNode, s1, s2 reflect.Value, changes *[]Chan continue } - node := structpath.NewStructField(path, sf.Tag, sf.Name) + jsonTag := structtag.JSONTag(sf.Tag.Get("json")) + + // Resolve field name from JSON tag or fall back to Go field name + fieldName := jsonTag.Name() + if fieldName == "" { + fieldName = sf.Name + } + node := structpath.NewStructField(path, fieldName) + v1Field := s1.Field(i) v2Field := s2.Field(i) @@ -135,7 +143,6 @@ func diffStruct(path *structpath.PathNode, s1, s2 reflect.Value, changes *[]Chan zero2 := v2Field.IsZero() if zero1 || zero2 { - jsonTag := structtag.JSONTag(sf.Tag.Get("json")) if jsonTag.OmitEmpty() { if zero1 { if !slices.Contains(forced1, sf.Name) { diff --git a/libs/structs/structpath/path.go b/libs/structs/structpath/path.go index dd12b62972..25d411cedd 100644 --- a/libs/structs/structpath/path.go +++ b/libs/structs/structpath/path.go @@ -3,18 +3,15 @@ package structpath import ( "errors" "fmt" - "reflect" "strconv" "strings" - - "github.com/databricks/cli/libs/structs/structtag" ) const ( - tagStruct = -1 - tagMapKey = -2 - tagAnyKey = -4 - tagAnyIndex = -5 + tagStruct = -1 + tagMapKey = -2 + tagDotStar = -4 + tagBracketStar = -5 ) // PathNode represents a node in a path for struct diffing. @@ -51,18 +48,18 @@ func (p *PathNode) MapKey() (string, bool) { return "", false } -func (p *PathNode) AnyKey() bool { +func (p *PathNode) DotStar() bool { if p == nil { return false } - return p.index == tagAnyKey + return p.index == tagDotStar } -func (p *PathNode) AnyIndex() bool { +func (p *PathNode) BracketStar() bool { if p == nil { return false } - return p.index == tagAnyIndex + return p.index == tagBracketStar } func (p *PathNode) Field() (string, bool) { @@ -82,6 +79,34 @@ func (p *PathNode) Parent() *PathNode { return p.prev } +// AsSlice returns the path as a slice of PathNodes from root to current. +// Efficiently pre-allocates the exact length and fills in reverse order. +func (p *PathNode) AsSlice() []*PathNode { + if p == nil { + return nil + } + + // First pass: count the length + length := 0 + current := p + for current != nil { + length++ + current = current.Parent() + } + + // Allocate slice with exact capacity + segments := make([]*PathNode, length) + + // Second pass: fill in reverse order (from end to start) + current = p + for i := length - 1; i >= 0; i-- { + segments[i] = current + current = current.Parent() + } + + return segments +} + // NewIndex creates a new PathNode for an array/slice index. func NewIndex(prev *PathNode, index int) *PathNode { if index < 0 { @@ -103,33 +128,26 @@ func NewMapKey(prev *PathNode, key string) *PathNode { } // NewStructField creates a new PathNode for a struct field. -// The jsonTag is used for JSON key resolution, and fieldName is used as fallback. -func NewStructField(prev *PathNode, tag reflect.StructTag, fieldName string) *PathNode { - jsonTag := structtag.JSONTag(tag.Get("json")) - - key := fieldName - if name := jsonTag.Name(); name != "" { - key = name - } - +// The fieldName should be the resolved field name (e.g., from JSON tag or Go field name). +func NewStructField(prev *PathNode, fieldName string) *PathNode { return &PathNode{ prev: prev, - key: key, + key: fieldName, index: tagStruct, } } -func NewAnyKey(prev *PathNode) *PathNode { +func NewDotStar(prev *PathNode) *PathNode { return &PathNode{ prev: prev, - index: tagAnyKey, + index: tagDotStar, } } -func NewAnyIndex(prev *PathNode) *PathNode { +func NewBracketStar(prev *PathNode) *PathNode { return &PathNode{ prev: prev, - index: tagAnyIndex, + index: tagBracketStar, } } @@ -149,8 +167,20 @@ func (p *PathNode) String() string { return p.prev.String() + "[" + strconv.Itoa(p.index) + "]" } - if p.index == tagAnyKey || p.index == tagAnyIndex { - return p.prev.String() + "[*]" + if p.index == tagDotStar { + prev := p.prev.String() + if prev == "" { + return "*" + } + return prev + ".*" + } + + if p.index == tagBracketStar { + prev := p.prev.String() + if prev == "" { + return "[*]" + } + return prev + "[*]" } if p.index == tagStruct { @@ -240,11 +270,11 @@ func Parse(s string) (*PathNode, error) { case stateField: if ch == '.' { - result = NewStructField(result, reflect.StructTag(""), currentToken.String()) + result = NewStructField(result, currentToken.String()) currentToken.Reset() state = stateFieldStart } else if ch == '[' { - result = NewStructField(result, reflect.StructTag(""), currentToken.String()) + result = NewStructField(result, currentToken.String()) currentToken.Reset() state = stateBracketOpen } else if !isReservedFieldChar(ch) { @@ -305,9 +335,7 @@ func Parse(s string) (*PathNode, error) { case stateWildcard: if ch == ']' { - // Note, since we're parsing this without type info present, we don't know if it's AnyKey or AnyIndex - // Perhaps structpath should be simplified to have Wildcard as merged representation of AnyKey/AnyIndex - result = NewAnyKey(result) + result = NewBracketStar(result) state = stateExpectDotOrEnd } else { return nil, fmt.Errorf("unexpected character '%c' after '*' at position %d", ch, pos) @@ -338,7 +366,7 @@ func Parse(s string) (*PathNode, error) { case stateStart: return result, nil // Empty path, result is nil case stateField: - result = NewStructField(result, reflect.StructTag(""), currentToken.String()) + result = NewStructField(result, currentToken.String()) return result, nil case stateExpectDotOrEnd: return result, nil @@ -380,35 +408,3 @@ func isReservedFieldChar(ch byte) bool { return false } } - -// Path in libs/dyn format -func (p *PathNode) DynPath() string { - if p == nil { - return "" - } - - if p.index >= 0 { - return p.prev.DynPath() + "[" + strconv.Itoa(p.index) + "]" - } - - if p.index == tagAnyKey { - prev := p.prev.DynPath() - if prev == "" { - return "*" - } else { - return prev + ".*" - } - } - - if p.index == tagAnyIndex { - return p.prev.DynPath() + "[*]" - } - - // Both struct fields and map keys use dot notation in DynPath - prev := p.prev.DynPath() - if prev == "" { - return p.key - } else { - return prev + "." + p.key - } -} diff --git a/libs/structs/structpath/path_test.go b/libs/structs/structpath/path_test.go index a2be306a94..0bb2543be1 100644 --- a/libs/structs/structpath/path_test.go +++ b/libs/structs/structpath/path_test.go @@ -1,7 +1,6 @@ package structpath import ( - "reflect" "testing" "github.com/stretchr/testify/assert" @@ -9,17 +8,15 @@ import ( func TestPathNode(t *testing.T) { tests := []struct { - name string - node *PathNode - String string - DynPath string // Only set when different from String - IgnoreDynPath bool // Do not test DynPath - Index any - MapKey any - Field any - Root any - AnyKey bool - AnyIndex bool + name string + node *PathNode + String string + Index any + MapKey any + Field any + Root any + DotStar bool + BracketStar bool }{ // Single node tests { @@ -35,137 +32,127 @@ func TestPathNode(t *testing.T) { Index: 5, }, { - name: "map key", - node: NewMapKey(nil, "mykey"), - String: `['mykey']`, - DynPath: "mykey", - MapKey: "mykey", + name: "map key", + node: NewMapKey(nil, "mykey"), + String: `['mykey']`, + MapKey: "mykey", }, { name: "struct field with JSON tag", - node: NewStructField(nil, reflect.StructTag(`json:"json_name"`), "GoFieldName"), + node: NewStructField(nil, "json_name"), String: "json_name", Field: "json_name", }, { name: "struct field without JSON tag (fallback to Go name)", - node: NewStructField(nil, reflect.StructTag(""), "GoFieldName"), + node: NewStructField(nil, "GoFieldName"), String: "GoFieldName", Field: "GoFieldName", }, { name: "struct field with dash JSON tag", - node: NewStructField(nil, reflect.StructTag(`json:"-"`), "GoFieldName"), + node: NewStructField(nil, "-"), String: "-", Field: "-", }, { name: "struct field with JSON tag options", - node: NewStructField(nil, reflect.StructTag(`json:"lazy_field,omitempty"`), "LazyField"), + node: NewStructField(nil, "lazy_field"), String: "lazy_field", Field: "lazy_field", }, { - name: "any key", - node: NewAnyKey(nil), - String: "[*]", - DynPath: "*", - AnyKey: true, + name: "dot star", + node: NewDotStar(nil), + String: "*", + DotStar: true, }, { - name: "any index", - node: NewAnyIndex(nil), - String: "[*]", - AnyIndex: true, + name: "bracket star", + node: NewBracketStar(nil), + String: "[*]", + BracketStar: true, }, // Two node tests { name: "struct field -> array index", - node: NewIndex(NewStructField(nil, reflect.StructTag(`json:"items"`), "Items"), 3), + node: NewIndex(NewStructField(nil, "items"), 3), String: "items[3]", Index: 3, }, { - name: "struct field -> map key", - node: NewMapKey(NewStructField(nil, reflect.StructTag(`json:"config"`), "Config"), "database"), - String: `config['database']`, - DynPath: "config.database", - MapKey: "database", + name: "struct field -> map key", + node: NewMapKey(NewStructField(nil, "config"), "database"), + String: `config['database']`, + MapKey: "database", }, { name: "struct field -> struct field", - node: NewStructField(NewStructField(nil, reflect.StructTag(`json:"user"`), "User"), reflect.StructTag(`json:"name"`), "Name"), + node: NewStructField(NewStructField(nil, "user"), "name"), String: "user.name", Field: "name", }, { - name: "map key -> array index", - node: NewIndex(NewMapKey(nil, "servers"), 0), - String: `['servers'][0]`, - DynPath: "servers[0]", - Index: 0, + name: "map key -> array index", + node: NewIndex(NewMapKey(nil, "servers"), 0), + String: `['servers'][0]`, + Index: 0, }, { - name: "map key -> struct field", - node: NewStructField(NewMapKey(nil, "primary"), reflect.StructTag(`json:"host"`), "Host"), - String: `['primary'].host`, - DynPath: `primary.host`, - Field: "host", + name: "map key -> struct field", + node: NewStructField(NewMapKey(nil, "primary"), "host"), + String: `['primary'].host`, + Field: "host", }, { name: "array index -> struct field", - node: NewStructField(NewIndex(nil, 2), reflect.StructTag(`json:"id"`), "ID"), + node: NewStructField(NewIndex(nil, 2), "id"), String: "[2].id", Field: "id", }, { - name: "array index -> map key", - node: NewMapKey(NewIndex(nil, 1), "status"), - String: `[1]['status']`, - DynPath: "[1].status", - MapKey: "status", + name: "array index -> map key", + node: NewMapKey(NewIndex(nil, 1), "status"), + String: `[1]['status']`, + MapKey: "status", }, { name: "struct field without JSON tag -> struct field with JSON tag", - node: NewStructField(NewStructField(nil, reflect.StructTag(""), "Parent"), reflect.StructTag(`json:"child_name"`), "ChildName"), + node: NewStructField(NewStructField(nil, "Parent"), "child_name"), String: "Parent.child_name", Field: "child_name", }, { - name: "any key", - node: NewAnyKey(NewStructField(nil, reflect.StructTag(""), "Parent")), - String: "Parent[*]", - DynPath: "Parent.*", - AnyKey: true, + name: "dot star with parent", + node: NewDotStar(NewStructField(nil, "Parent")), + String: "Parent.*", + DotStar: true, }, { - name: "any index", - node: NewAnyIndex(NewStructField(nil, reflect.StructTag(""), "Parent")), - String: "Parent[*]", - AnyIndex: true, + name: "bracket star with parent", + node: NewBracketStar(NewStructField(nil, "Parent")), + String: "Parent[*]", + BracketStar: true, }, // Edge cases with special characters in map keys { - name: "map key with single quote", - node: NewMapKey(nil, "key's"), - String: `['key''s']`, - DynPath: "key's", - MapKey: "key's", + name: "map key with single quote", + node: NewMapKey(nil, "key's"), + String: `['key''s']`, + MapKey: "key's", }, { - name: "map key with multiple single quotes", - node: NewMapKey(nil, "''"), - String: `['''''']`, - DynPath: "''", - MapKey: "''", + name: "map key with multiple single quotes", + node: NewMapKey(nil, "''"), + String: `['''''']`, + MapKey: "''", }, { - name: "empty map key", - node: NewMapKey(nil, ""), - String: `['']`, - IgnoreDynPath: true, - MapKey: "", + name: "empty map key", + node: NewMapKey(nil, ""), + String: `['']`, + MapKey: "", }, { name: "complex path", @@ -173,45 +160,43 @@ func TestPathNode(t *testing.T) { NewIndex( NewMapKey( NewStructField( - NewStructField(nil, reflect.StructTag(`json:"user"`), "User"), - reflect.StructTag(`json:"settings"`), "Settings"), + NewStructField(nil, "user"), + "settings"), "theme"), 0), - reflect.StructTag(`json:"color"`), "Color"), - String: "user.settings['theme'][0].color", - DynPath: "user.settings.theme[0].color", - Field: "color", + "color"), + String: "user.settings['theme'][0].color", + Field: "color", }, { name: "field with special characters", - node: NewStructField(nil, reflect.StructTag(""), "field@name:with#symbols!"), + node: NewStructField(nil, "field@name:with#symbols!"), String: "field@name:with#symbols!", Field: "field@name:with#symbols!", }, { name: "field with spaces", - node: NewStructField(nil, reflect.StructTag(""), "field with spaces"), + node: NewStructField(nil, "field with spaces"), String: "field with spaces", Field: "field with spaces", }, { name: "field starting with digit", - node: NewStructField(nil, reflect.StructTag(""), "123field"), + node: NewStructField(nil, "123field"), String: "123field", Field: "123field", }, { name: "field with unicode", - node: NewStructField(nil, reflect.StructTag(""), "名前🙂"), + node: NewStructField(nil, "名前🙂"), String: "名前🙂", Field: "名前🙂", }, { - name: "map key with reserved characters", - node: NewMapKey(nil, "key\x00[],`"), - String: "['key\x00[],`']", - DynPath: "key\x00[],`", - MapKey: "key\x00[],`", + name: "map key with reserved characters", + node: NewMapKey(nil, "key\x00[],`"), + String: "['key\x00[],`']", + MapKey: "key\x00[],`", }, } @@ -229,17 +214,6 @@ func TestPathNode(t *testing.T) { assert.Equal(t, tt.String, roundtripResult, "Roundtrip conversion should be identical") } - if !tt.IgnoreDynPath { - dynResult := tt.node.DynPath() - expectedDyn := tt.String - if tt.DynPath != "" { - expectedDyn = tt.DynPath - // Enforce rule: DynPath should only be set when different from String - assert.NotEqual(t, expectedDyn, tt.String, "Test case %q: DynPath should only be set when different from String", tt.name) - } - assert.Equal(t, expectedDyn, dynResult, "DynPath() method") - } - // Index gotIndex, isIndex := tt.node.Index() if tt.Index == nil { @@ -281,9 +255,9 @@ func TestPathNode(t *testing.T) { assert.True(t, isRoot) } - // AnyKey, AnyIndex - assert.Equal(t, tt.AnyKey, tt.node.AnyKey()) - assert.Equal(t, tt.AnyIndex, tt.node.AnyIndex()) + // DotStar and BracketStar + assert.Equal(t, tt.DotStar, tt.node.DotStar()) + assert.Equal(t, tt.BracketStar, tt.node.BracketStar()) }) } } diff --git a/libs/structs/structwalk/walk.go b/libs/structs/structwalk/walk.go index 4eeaed348f..4acf6d2ba0 100644 --- a/libs/structs/structwalk/walk.go +++ b/libs/structs/structwalk/walk.go @@ -115,12 +115,18 @@ func walkStruct(path *structpath.PathNode, s reflect.Value, visit VisitFunc) { continue } - node := structpath.NewStructField(path, sf.Tag, sf.Name) jsonTag := structtag.JSONTag(sf.Tag.Get("json")) if jsonTag.Name() == "-" { continue // skip fields without json name } + // Resolve field name from JSON tag or fall back to Go field name + fieldName := jsonTag.Name() + if fieldName == "" { + fieldName = sf.Name + } + node := structpath.NewStructField(path, fieldName) + fieldVal := s.Field(i) // Skip zero values with omitempty unless field is explicitly forced. if jsonTag.OmitEmpty() && fieldVal.IsZero() && !slices.Contains(forced, sf.Name) { diff --git a/libs/structs/structwalk/walktype.go b/libs/structs/structwalk/walktype.go index d1f5a361af..8d9ddcf20b 100644 --- a/libs/structs/structwalk/walktype.go +++ b/libs/structs/structwalk/walktype.go @@ -84,14 +84,14 @@ func walkTypeValue(path *structpath.PathNode, typ reflect.Type, field *reflect.S walkTypeStruct(path, typ, visit, visitedCount) case reflect.Slice, reflect.Array: - walkTypeValue(structpath.NewAnyIndex(path), typ.Elem(), nil, visit, visitedCount) + walkTypeValue(structpath.NewBracketStar(path), typ.Elem(), nil, visit, visitedCount) case reflect.Map: if typ.Key().Kind() != reflect.String { return // unsupported map key type } // For maps, we walk the value type directly at the current path - walkTypeValue(structpath.NewAnyKey(path), typ.Elem(), nil, visit, visitedCount) + walkTypeValue(structpath.NewBracketStar(path), typ.Elem(), nil, visit, visitedCount) default: // func, chan, interface, invalid, etc. -> ignore @@ -122,7 +122,12 @@ func walkTypeStruct(path *structpath.PathNode, st reflect.Type, visit VisitTypeF continue } - node := structpath.NewStructField(path, sf.Tag, sf.Name) + // Resolve field name from JSON tag or fall back to Go field name + fieldName := jsonTagName + if fieldName == "" { + fieldName = sf.Name + } + node := structpath.NewStructField(path, fieldName) walkTypeValue(node, sf.Type, &sf, visit, visitedCount) } } diff --git a/libs/structs/structwalk/walktype_test.go b/libs/structs/structwalk/walktype_test.go index 4b1ce085b0..f49496f26c 100644 --- a/libs/structs/structwalk/walktype_test.go +++ b/libs/structs/structwalk/walktype_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/libs/structs/dynpath" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" @@ -29,8 +30,7 @@ func getScalarFields(t *testing.T, typ reflect.Type) map[string]any { pathNew, err := structpath.Parse(s) if assert.NoError(t, err, "Parse(path.String()) failed for %q: %s", s, err) { newS := pathNew.String() - // This still does not work because of AnyKey / AnyIndex ambiguity - // assert.Equal(t, path, pathNew, "Parse(path.String()) returned different path;\npath=%#v %q\npathNew=%#v %q", path, s, pathNew, newS) + assert.Equal(t, path, pathNew, "Parse(path.String()) returned different path;\npath=%#v %q\npathNew=%#v %q", path, s, pathNew, newS) assert.Equal(t, s, newS, "Parse(path.String()).String() is different from path.String()\npath.String()=%q\npathNew.String()=%q", path, pathNew) } @@ -166,15 +166,15 @@ func TestTypeRoot(t *testing.T) { ) } -func getReadonlyFields(t *testing.T, typ reflect.Type) []string { +func getReadonlyFields(t *testing.T, rootType reflect.Type) []string { var results []string - err := WalkType(typ, func(path *structpath.PathNode, typ reflect.Type, field *reflect.StructField) (continueWalk bool) { + err := WalkType(rootType, func(path *structpath.PathNode, typ reflect.Type, field *reflect.StructField) (continueWalk bool) { if path == nil || field == nil { return true } bundleTag := field.Tag.Get("bundle") if strings.Contains(bundleTag, "readonly") { - results = append(results, path.DynPath()) + results = append(results, dynpath.ConvertPathNodeToDynPath(path, rootType)) } return true }) From 8d957ad7f65cd8e51fde9df179a83c80befdfbcc Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 19 Sep 2025 11:14:11 +0200 Subject: [PATCH 02/20] wip --- libs/structs/dynpath/dynpath.go | 3 + libs/structs/dynpath/dynpath_test.go | 135 +++++++++++++-------------- 2 files changed, 67 insertions(+), 71 deletions(-) diff --git a/libs/structs/dynpath/dynpath.go b/libs/structs/dynpath/dynpath.go index 50744919c1..2fcacb7b78 100644 --- a/libs/structs/dynpath/dynpath.go +++ b/libs/structs/dynpath/dynpath.go @@ -1,6 +1,8 @@ package dynpath import ( + "fmt" + "os" "reflect" "strconv" "strings" @@ -18,6 +20,7 @@ func ConvertPathNodeToDynPath(path *structpath.PathNode, rootType reflect.Type) if path == nil { return "" } + fmt.Fprintf(os.Stderr, "pathg=%#v\n", path) segments := path.AsSlice() var result strings.Builder diff --git a/libs/structs/dynpath/dynpath_test.go b/libs/structs/dynpath/dynpath_test.go index 0a89e6d966..8dcbc3265c 100644 --- a/libs/structs/dynpath/dynpath_test.go +++ b/libs/structs/dynpath/dynpath_test.go @@ -32,15 +32,10 @@ func TestConvertPathNodeToDynPath(t *testing.T) { dynPath: "mykey", }, { - name: "struct field with JSON tag", + name: "struct field", structPath: "json_name", dynPath: "json_name", }, - { - name: "struct field without JSON tag", - structPath: "GoFieldName", - dynPath: "GoFieldName", - }, { name: "dot star", structPath: "*", @@ -60,41 +55,42 @@ func TestConvertPathNodeToDynPath(t *testing.T) { }, // Compound paths - { - name: "struct field -> array index", - structPath: "items[3]", - dynPath: "items[3]", - }, - { - name: "struct field -> map key", - structPath: "config['database']", - dynPath: "config.database", - }, - { - name: "struct field -> struct field", - structPath: "user.name", - dynPath: "user.name", - }, - { - name: "map key -> array index", - structPath: "['servers'][0]", - dynPath: "servers[0]", - }, - { - name: "map key -> struct field", - structPath: "['primary'].host", - dynPath: "primary.host", - }, - { - name: "array index -> struct field", - structPath: "[2].id", - dynPath: "[2].id", - }, - { - name: "array index -> map key", - structPath: "[1]['status']", - dynPath: "[1].status", - }, + /* + { + name: "struct field -> array index", + structPath: "items[3]", + dynPath: "items[3]", + }, + { + name: "struct field -> map key", + structPath: "config['database']", + dynPath: "config.database", + }, + { + name: "struct field -> struct field", + structPath: "user.name", + dynPath: "user.name", + }, + { + name: "map key -> array index", + structPath: "['servers'][0]", + dynPath: "servers[0]", + }, + { + name: "map key -> struct field", + structPath: "['primary'].host", + dynPath: "primary.host", + }, + { + name: "array index -> struct field", + structPath: "[2].id", + dynPath: "[2].id", + }, + { + name: "array index -> map key", + structPath: "[1]['status']", + dynPath: "[1].status", + },*/ // Wildcard combinations { @@ -155,44 +151,41 @@ func TestConvertPathNodeToDynPath(t *testing.T) { // Complex real-world example { name: "complex nested path", - structPath: "user.settings['theme'][0].color", - dynPath: "user.settings.theme[0].color", + structPath: "user.*.settings['theme'][0].color", + dynPath: "user.*.settings.theme[0].color", }, // Mixed JSON tag and Go field name scenarios - { - name: "Go field -> JSON field", - structPath: "Parent.child_name", - dynPath: "Parent.child_name", - }, - { - name: "JSON field -> Go field", - structPath: "parent.ChildName", - dynPath: "parent.ChildName", - }, - { - name: "dash JSON tag field", - structPath: "-", - dynPath: "-", - }, - { - name: "JSON tag with options", - structPath: "lazy_field", - dynPath: "lazy_field", - }, + //{ + // name: "Go field -> JSON field", + // structPath: "Parent.child_name", + // dynPath: "Parent.child_name", + //}, + //{ + // name: "JSON field -> Go field", + // structPath: "parent.ChildName", + // dynPath: "parent.ChildName", + //}, + //{ + // name: "dash JSON tag field", + // structPath: "-", + // dynPath: "-", + //}, + //{ + // name: "JSON tag with options", + // structPath: "lazy_field", + // dynPath: "lazy_field", + //}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - // Parse the structPath string into a PathNode - var pathNode *structpath.PathNode - var err error - if tt.structPath == "" { - pathNode = nil - } else { - pathNode, err = structpath.Parse(tt.structPath) - assert.NoError(t, err, "Failed to parse structPath: %s", tt.structPath) + if tt.name != "dot star with parent" { + return } + pathNode, err := structpath.Parse(tt.structPath) + t.Logf("%q node=%#v", tt.structPath, pathNode) + assert.NoError(t, err, "Failed to parse structPath: %s", tt.structPath) // Convert to DynPath and verify result result := ConvertPathNodeToDynPath(pathNode, tt.rootType) From 8b1d9f7f00370fe0b60abc6d697284fc9be2db86 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 19 Sep 2025 11:38:08 +0200 Subject: [PATCH 03/20] fixes --- libs/structs/dynpath/dynpath.go | 23 ++++++--- libs/structs/dynpath/dynpath_test.go | 74 +++++++++++++--------------- libs/structs/structpath/path.go | 31 ++++++++++-- libs/structs/structpath/path_test.go | 38 +++++++++++++- 4 files changed, 112 insertions(+), 54 deletions(-) diff --git a/libs/structs/dynpath/dynpath.go b/libs/structs/dynpath/dynpath.go index 2fcacb7b78..eb7fbd5f4e 100644 --- a/libs/structs/dynpath/dynpath.go +++ b/libs/structs/dynpath/dynpath.go @@ -77,25 +77,32 @@ func ConvertPathNodeToDynPath(path *structpath.PathNode, rootType reflect.Type) result.WriteString(".") } result.WriteString("*") - // Type doesn't change for wildcards - } else if segment.BracketStar() { - // Context-aware wildcard rendering + // If it's a map, we can inspect the type; otherwise we cannot since we don't know what field to look into if currentType != nil && currentType.Kind() == reflect.Map { + currentType = currentType.Elem() + } else { + currentType = nil + } + + } else if segment.BracketStar() { + if currentType != nil && (currentType.Kind() == reflect.Array || currentType.Kind() == reflect.Slice) { + result.WriteString("[*]") + currentType = currentType.Elem() + } else { // Map wildcard uses dot notation in DynPath if result.Len() > 0 { result.WriteString(".") } result.WriteString("*") - currentType = currentType.Elem() - } else { - // Array/slice wildcard or fallback uses bracket notation - result.WriteString("[*]") - if currentType != nil && (currentType.Kind() == reflect.Array || currentType.Kind() == reflect.Slice) { + + if currentType != nil && currentType.Kind() == reflect.Map { currentType = currentType.Elem() } else { currentType = nil } + + // QQQ return error if we cannot disambiguate? } } } diff --git a/libs/structs/dynpath/dynpath_test.go b/libs/structs/dynpath/dynpath_test.go index 8dcbc3265c..1d038fa4f5 100644 --- a/libs/structs/dynpath/dynpath_test.go +++ b/libs/structs/dynpath/dynpath_test.go @@ -55,42 +55,41 @@ func TestConvertPathNodeToDynPath(t *testing.T) { }, // Compound paths - /* - { - name: "struct field -> array index", - structPath: "items[3]", - dynPath: "items[3]", - }, - { - name: "struct field -> map key", - structPath: "config['database']", - dynPath: "config.database", - }, - { - name: "struct field -> struct field", - structPath: "user.name", - dynPath: "user.name", - }, - { - name: "map key -> array index", - structPath: "['servers'][0]", - dynPath: "servers[0]", - }, - { - name: "map key -> struct field", - structPath: "['primary'].host", - dynPath: "primary.host", - }, - { - name: "array index -> struct field", - structPath: "[2].id", - dynPath: "[2].id", - }, - { - name: "array index -> map key", - structPath: "[1]['status']", - dynPath: "[1].status", - },*/ + { + name: "struct field -> array index", + structPath: "items[3]", + dynPath: "items[3]", + }, + { + name: "struct field -> map key", + structPath: "config['database']", + dynPath: "config.database", + }, + { + name: "struct field -> struct field", + structPath: "user.name", + dynPath: "user.name", + }, + { + name: "map key -> array index", + structPath: "['servers'][0]", + dynPath: "servers[0]", + }, + { + name: "map key -> struct field", + structPath: "['primary'].host", + dynPath: "primary.host", + }, + { + name: "array index -> struct field", + structPath: "[2].id", + dynPath: "[2].id", + }, + { + name: "array index -> map key", + structPath: "[1]['status']", + dynPath: "[1].status", + }, // Wildcard combinations { @@ -180,9 +179,6 @@ func TestConvertPathNodeToDynPath(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if tt.name != "dot star with parent" { - return - } pathNode, err := structpath.Parse(tt.structPath) t.Logf("%q node=%#v", tt.structPath, pathNode) assert.NoError(t, err, "Failed to parse structPath: %s", tt.structPath) diff --git a/libs/structs/structpath/path.go b/libs/structs/structpath/path.go index 25d411cedd..9c900b80be 100644 --- a/libs/structs/structpath/path.go +++ b/libs/structs/structpath/path.go @@ -201,9 +201,10 @@ func (p *PathNode) String() string { // State Machine for Path Parsing: // // States: -// - START: Beginning of parsing, expects field name or "[" -// - FIELD_START: After a dot, expects field name only +// - START: Beginning of parsing, expects field name, "[", or "*" +// - FIELD_START: After a dot, expects field name or "*" // - FIELD: Reading field name characters +// - DOT_STAR: Encountered "*" (at start or after dot), expects ".", "[", or EOF // - BRACKET_OPEN: Just encountered "[", expects digit, "'" or "*" // - INDEX: Reading array index digits, expects more digits or "]" // - MAP_KEY: Reading map key content, expects any char or "'" @@ -213,9 +214,10 @@ func (p *PathNode) String() string { // - END: Successfully completed parsing // // Transitions: -// - START: [a-zA-Z_-] -> FIELD, "[" -> BRACKET_OPEN, EOF -> END -// - FIELD_START: [a-zA-Z_-] -> FIELD, other -> ERROR +// - START: [a-zA-Z_-] -> FIELD, "[" -> BRACKET_OPEN, "*" -> DOT_STAR, EOF -> END +// - FIELD_START: [a-zA-Z_-] -> FIELD, "*" -> DOT_STAR, other -> ERROR // - FIELD: [a-zA-Z0-9_-] -> FIELD, "." -> FIELD_START, "[" -> BRACKET_OPEN, EOF -> END +// - DOT_STAR: "." -> FIELD_START, "[" -> BRACKET_OPEN, EOF -> END, other -> ERROR // - BRACKET_OPEN: [0-9] -> INDEX, "'" -> MAP_KEY, "*" -> WILDCARD // - INDEX: [0-9] -> INDEX, "]" -> EXPECT_DOT_OR_END // - MAP_KEY: (any except "'") -> MAP_KEY, "'" -> MAP_KEY_QUOTE @@ -232,6 +234,7 @@ func Parse(s string) (*PathNode, error) { stateStart = iota stateFieldStart stateField + stateDotStar stateBracketOpen stateIndex stateMapKey @@ -253,6 +256,8 @@ func Parse(s string) (*PathNode, error) { case stateStart: if ch == '[' { state = stateBracketOpen + } else if ch == '*' { + state = stateDotStar } else if !isReservedFieldChar(ch) { currentToken.WriteByte(ch) state = stateField @@ -261,7 +266,9 @@ func Parse(s string) (*PathNode, error) { } case stateFieldStart: - if !isReservedFieldChar(ch) { + if ch == '*' { + state = stateDotStar + } else if !isReservedFieldChar(ch) { currentToken.WriteByte(ch) state = stateField } else { @@ -283,6 +290,17 @@ func Parse(s string) (*PathNode, error) { return nil, fmt.Errorf("invalid character '%c' in field name at position %d", ch, pos) } + case stateDotStar: + if ch == '.' { + result = NewDotStar(result) + state = stateFieldStart + } else if ch == '[' { + result = NewDotStar(result) + state = stateBracketOpen + } else { + return nil, fmt.Errorf("unexpected character '%c' after '.*' at position %d", ch, pos) + } + case stateBracketOpen: if ch >= '0' && ch <= '9' { currentToken.WriteByte(ch) @@ -368,6 +386,9 @@ func Parse(s string) (*PathNode, error) { case stateField: result = NewStructField(result, currentToken.String()) return result, nil + case stateDotStar: + result = NewDotStar(result) + return result, nil case stateExpectDotOrEnd: return result, nil case stateFieldStart: diff --git a/libs/structs/structpath/path_test.go b/libs/structs/structpath/path_test.go index 0bb2543be1..a833e5611e 100644 --- a/libs/structs/structpath/path_test.go +++ b/libs/structs/structpath/path_test.go @@ -135,6 +135,7 @@ func TestPathNode(t *testing.T) { String: "Parent[*]", BracketStar: true, }, + // Edge cases with special characters in map keys { name: "map key with single quote", @@ -198,6 +199,32 @@ func TestPathNode(t *testing.T) { String: "['key\x00[],`']", MapKey: "key\x00[],`", }, + + // Additional dot-star pattern tests + { + name: "field dot star", + node: NewDotStar(NewStructField(nil, "bla")), + String: "bla.*", + DotStar: true, + }, + { + name: "field dot star dot field", + node: NewStructField(NewDotStar(NewStructField(nil, "bla")), "foo"), + String: "bla.*.foo", + Field: "foo", + }, + { + name: "field dot star bracket index", + node: NewIndex(NewDotStar(NewStructField(nil, "bla")), 0), + String: "bla.*[0]", + Index: 0, + }, + { + name: "field dot star bracket star", + node: NewBracketStar(NewDotStar(NewStructField(nil, "bla"))), + String: "bla.*[*]", + BracketStar: true, + }, } for _, tt := range tests { @@ -208,8 +235,8 @@ func TestPathNode(t *testing.T) { // Test roundtrip conversion: String() -> Parse() -> String() parsed, err := Parse(tt.String) - assert.NoError(t, err, "Parse() should not error") - if parsed != nil { + if assert.NoError(t, err, "Parse() should not error") { + assert.Equal(t, tt.node, parsed) roundtripResult := parsed.String() assert.Equal(t, tt.String, roundtripResult, "Roundtrip conversion should be identical") } @@ -408,6 +435,13 @@ func TestParseErrors(t *testing.T) { input: "field['key'", error: "unexpected end of input after quote in map key", }, + + // Invalid dot-star patterns + { + name: "dot star followed by field name", + input: "bla.*foo", + error: "unexpected character 'f' after '.*' at position 5", + }, } for _, tt := range tests { From d609aacfd8e27bed3c3cb22c3227b747b6043095 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 19 Sep 2025 11:46:42 +0200 Subject: [PATCH 04/20] clean up --- libs/structs/dynpath/dynpath.go | 7 ------- libs/structs/dynpath/dynpath_test.go | 22 ++++++++++++++++++---- libs/structs/structpath/path.go | 4 ---- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/libs/structs/dynpath/dynpath.go b/libs/structs/dynpath/dynpath.go index eb7fbd5f4e..d8901e657f 100644 --- a/libs/structs/dynpath/dynpath.go +++ b/libs/structs/dynpath/dynpath.go @@ -1,8 +1,6 @@ package dynpath import ( - "fmt" - "os" "reflect" "strconv" "strings" @@ -17,11 +15,6 @@ import ( // - BracketStar accessing arrays/slices renders as "parent[*]" // - DotStar always renders as "parent.*" func ConvertPathNodeToDynPath(path *structpath.PathNode, rootType reflect.Type) string { - if path == nil { - return "" - } - fmt.Fprintf(os.Stderr, "pathg=%#v\n", path) - segments := path.AsSlice() var result strings.Builder currentType := rootType diff --git a/libs/structs/dynpath/dynpath_test.go b/libs/structs/dynpath/dynpath_test.go index 1d038fa4f5..4e0bb4c3ae 100644 --- a/libs/structs/dynpath/dynpath_test.go +++ b/libs/structs/dynpath/dynpath_test.go @@ -100,14 +100,28 @@ func TestConvertPathNodeToDynPath(t *testing.T) { { name: "bracket star with parent - array", structPath: "Parent[*]", - rootType: reflect.TypeOf(struct{ Parent []int }{}), - dynPath: "Parent[*]", + rootType: reflect.TypeOf(struct { + Parent []int + }{}), + dynPath: "Parent[*]", }, + + { + name: "bracket star with parent - array", + structPath: "parent[*]", + rootType: reflect.TypeOf(struct { + Parent []int `json:"parent"` + }{}), + dynPath: "parent[*]", + }, + { name: "bracket star with parent - map", structPath: "Parent[*]", - rootType: reflect.TypeOf(struct{ Parent map[string]int }{}), - dynPath: "Parent.*", + rootType: reflect.TypeOf(struct { + Parent map[string]int + }{}), + dynPath: "Parent.*", }, // Special characters and edge cases diff --git a/libs/structs/structpath/path.go b/libs/structs/structpath/path.go index 9c900b80be..04f49cfc2d 100644 --- a/libs/structs/structpath/path.go +++ b/libs/structs/structpath/path.go @@ -82,10 +82,6 @@ func (p *PathNode) Parent() *PathNode { // AsSlice returns the path as a slice of PathNodes from root to current. // Efficiently pre-allocates the exact length and fills in reverse order. func (p *PathNode) AsSlice() []*PathNode { - if p == nil { - return nil - } - // First pass: count the length length := 0 current := p From 538492dd3c6e21cc90eb82c1f9f7d8905ef3d257 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 19 Sep 2025 12:02:24 +0200 Subject: [PATCH 05/20] lint fix --- libs/structs/structpath/path.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libs/structs/structpath/path.go b/libs/structs/structpath/path.go index 04f49cfc2d..4c4ae2daef 100644 --- a/libs/structs/structpath/path.go +++ b/libs/structs/structpath/path.go @@ -287,13 +287,14 @@ func Parse(s string) (*PathNode, error) { } case stateDotStar: - if ch == '.' { + switch ch { + case '.': result = NewDotStar(result) state = stateFieldStart - } else if ch == '[' { + case '[': result = NewDotStar(result) state = stateBracketOpen - } else { + default: return nil, fmt.Errorf("unexpected character '%c' after '.*' at position %d", ch, pos) } From affa4c92e76fb82ccf1080e17e4ae8447c674776 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 22 Sep 2025 14:06:31 +0200 Subject: [PATCH 06/20] fix walk --- bundle/internal/validation/enum.go | 4 +- bundle/internal/validation/required.go | 3 +- libs/structs/dynpath/dynpath.go | 104 ------------ libs/structs/dynpath/dynpath_test.go | 205 ----------------------- libs/structs/structpath/path.go | 40 ++++- libs/structs/structpath/path_test.go | 8 +- libs/structs/structwalk/walk.go | 2 +- libs/structs/structwalk/walktype.go | 4 +- libs/structs/structwalk/walktype_test.go | 51 +++--- 9 files changed, 73 insertions(+), 348 deletions(-) delete mode 100644 libs/structs/dynpath/dynpath.go delete mode 100644 libs/structs/dynpath/dynpath_test.go diff --git a/bundle/internal/validation/enum.go b/bundle/internal/validation/enum.go index d48cfa9dc2..4b1704da97 100644 --- a/bundle/internal/validation/enum.go +++ b/bundle/internal/validation/enum.go @@ -11,7 +11,6 @@ import ( "text/template" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/structs/dynpath" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/structs/structtag" "github.com/databricks/cli/libs/structs/structwalk" @@ -133,8 +132,7 @@ func extractEnumFields(typ reflect.Type) ([]EnumPatternInfo, error) { return true } - fieldPath := dynpath.ConvertPathNodeToDynPath(path, typ) - fieldsByPattern[fieldPath] = enumValues + fieldsByPattern[path.String()] = enumValues } return true }) diff --git a/bundle/internal/validation/required.go b/bundle/internal/validation/required.go index 8de2f0e589..d40e149d65 100644 --- a/bundle/internal/validation/required.go +++ b/bundle/internal/validation/required.go @@ -11,7 +11,6 @@ import ( "text/template" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/structs/dynpath" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/structs/structtag" "github.com/databricks/cli/libs/structs/structwalk" @@ -69,7 +68,7 @@ func extractRequiredFields(typ reflect.Type) ([]RequiredPatternInfo, error) { return true } - parentPath := dynpath.ConvertPathNodeToDynPath(path.Parent(), typ) + parentPath := path.Parent().String() fieldsByPattern[parentPath] = append(fieldsByPattern[parentPath], fieldName) return true }) diff --git a/libs/structs/dynpath/dynpath.go b/libs/structs/dynpath/dynpath.go deleted file mode 100644 index d8901e657f..0000000000 --- a/libs/structs/dynpath/dynpath.go +++ /dev/null @@ -1,104 +0,0 @@ -package dynpath - -import ( - "reflect" - "strconv" - "strings" - - "github.com/databricks/cli/libs/structs/structaccess" - "github.com/databricks/cli/libs/structs/structpath" -) - -// ConvertPathNodeToDynPath converts a PathNode to dyn path format string. -// Uses the provided root type to determine context-aware wildcard formatting: -// - BracketStar accessing maps renders as "parent.*" -// - BracketStar accessing arrays/slices renders as "parent[*]" -// - DotStar always renders as "parent.*" -func ConvertPathNodeToDynPath(path *structpath.PathNode, rootType reflect.Type) string { - segments := path.AsSlice() - var result strings.Builder - currentType := rootType - - for _, segment := range segments { - // Dereference pointers - for currentType != nil && currentType.Kind() == reflect.Pointer { - currentType = currentType.Elem() - } - - if index, ok := segment.Index(); ok { - // Array/slice index access - result.WriteString("[") - result.WriteString(strconv.Itoa(index)) - result.WriteString("]") - if currentType != nil && (currentType.Kind() == reflect.Array || currentType.Kind() == reflect.Slice) { - currentType = currentType.Elem() - } else { - currentType = nil - } - - } else if field, ok := segment.Field(); ok { - // Struct field access - if result.Len() > 0 { - result.WriteString(".") - } - result.WriteString(field) - if currentType != nil && currentType.Kind() == reflect.Struct { - if sf, _, ok := structaccess.FindStructFieldByKeyType(currentType, field); ok { - currentType = sf.Type - } else { - currentType = nil - } - } else { - currentType = nil - } - - } else if key, ok := segment.MapKey(); ok { - // Map key access - always use dot notation in DynPath - if result.Len() > 0 { - result.WriteString(".") - } - result.WriteString(key) - if currentType != nil && currentType.Kind() == reflect.Map { - currentType = currentType.Elem() - } else { - currentType = nil - } - - } else if segment.DotStar() { - // Field wildcard - always uses dot notation - if result.Len() > 0 { - result.WriteString(".") - } - result.WriteString("*") - - // If it's a map, we can inspect the type; otherwise we cannot since we don't know what field to look into - if currentType != nil && currentType.Kind() == reflect.Map { - currentType = currentType.Elem() - } else { - currentType = nil - } - - } else if segment.BracketStar() { - if currentType != nil && (currentType.Kind() == reflect.Array || currentType.Kind() == reflect.Slice) { - result.WriteString("[*]") - currentType = currentType.Elem() - } else { - // Map wildcard uses dot notation in DynPath - if result.Len() > 0 { - result.WriteString(".") - } - result.WriteString("*") - - if currentType != nil && currentType.Kind() == reflect.Map { - currentType = currentType.Elem() - } else { - currentType = nil - } - - // QQQ return error if we cannot disambiguate? - } - } - } - - return result.String() -} diff --git a/libs/structs/dynpath/dynpath_test.go b/libs/structs/dynpath/dynpath_test.go deleted file mode 100644 index 4e0bb4c3ae..0000000000 --- a/libs/structs/dynpath/dynpath_test.go +++ /dev/null @@ -1,205 +0,0 @@ -package dynpath - -import ( - "reflect" - "testing" - - "github.com/databricks/cli/libs/structs/structpath" - "github.com/stretchr/testify/assert" -) - -func TestConvertPathNodeToDynPath(t *testing.T) { - tests := []struct { - name string - structPath string // PathNode string representation to be parsed - dynPath string // Expected DynPath format - rootType reflect.Type // optional, for context-aware wildcard rendering - }{ - // Basic node types - { - name: "nil path", - structPath: "", - dynPath: "", - }, - { - name: "array index", - structPath: "[5]", - dynPath: "[5]", - }, - { - name: "map key", - structPath: "['mykey']", - dynPath: "mykey", - }, - { - name: "struct field", - structPath: "json_name", - dynPath: "json_name", - }, - { - name: "dot star", - structPath: "*", - dynPath: "*", - }, - { - name: "bracket star - array type", - structPath: "[*]", - rootType: reflect.TypeOf([]int{}), - dynPath: "[*]", - }, - { - name: "bracket star - map type", - structPath: "[*]", - rootType: reflect.TypeOf(map[string]int{}), - dynPath: "*", - }, - - // Compound paths - { - name: "struct field -> array index", - structPath: "items[3]", - dynPath: "items[3]", - }, - { - name: "struct field -> map key", - structPath: "config['database']", - dynPath: "config.database", - }, - { - name: "struct field -> struct field", - structPath: "user.name", - dynPath: "user.name", - }, - { - name: "map key -> array index", - structPath: "['servers'][0]", - dynPath: "servers[0]", - }, - { - name: "map key -> struct field", - structPath: "['primary'].host", - dynPath: "primary.host", - }, - { - name: "array index -> struct field", - structPath: "[2].id", - dynPath: "[2].id", - }, - { - name: "array index -> map key", - structPath: "[1]['status']", - dynPath: "[1].status", - }, - - // Wildcard combinations - { - name: "dot star with parent", - structPath: "Parent.*", - dynPath: "Parent.*", - }, - { - name: "bracket star with parent - array", - structPath: "Parent[*]", - rootType: reflect.TypeOf(struct { - Parent []int - }{}), - dynPath: "Parent[*]", - }, - - { - name: "bracket star with parent - array", - structPath: "parent[*]", - rootType: reflect.TypeOf(struct { - Parent []int `json:"parent"` - }{}), - dynPath: "parent[*]", - }, - - { - name: "bracket star with parent - map", - structPath: "Parent[*]", - rootType: reflect.TypeOf(struct { - Parent map[string]int - }{}), - dynPath: "Parent.*", - }, - - // Special characters and edge cases - { - name: "map key with single quote", - structPath: "['key''s']", - dynPath: "key's", - }, - { - name: "map key with multiple single quotes", - structPath: "['''''']", - dynPath: "''", - }, - { - name: "empty map key", - structPath: "['']", - dynPath: "", - }, - { - name: "map key with reserved characters", - structPath: "['key\x00[],`']", - dynPath: "key\x00[],`", - }, - { - name: "field with special characters", - structPath: "field@name:with#symbols!", - dynPath: "field@name:with#symbols!", - }, - { - name: "field with spaces", - structPath: "field with spaces", - dynPath: "field with spaces", - }, - { - name: "field with unicode", - structPath: "名前🙂", - dynPath: "名前🙂", - }, - - // Complex real-world example - { - name: "complex nested path", - structPath: "user.*.settings['theme'][0].color", - dynPath: "user.*.settings.theme[0].color", - }, - - // Mixed JSON tag and Go field name scenarios - //{ - // name: "Go field -> JSON field", - // structPath: "Parent.child_name", - // dynPath: "Parent.child_name", - //}, - //{ - // name: "JSON field -> Go field", - // structPath: "parent.ChildName", - // dynPath: "parent.ChildName", - //}, - //{ - // name: "dash JSON tag field", - // structPath: "-", - // dynPath: "-", - //}, - //{ - // name: "JSON tag with options", - // structPath: "lazy_field", - // dynPath: "lazy_field", - //}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - pathNode, err := structpath.Parse(tt.structPath) - t.Logf("%q node=%#v", tt.structPath, pathNode) - assert.NoError(t, err, "Failed to parse structPath: %s", tt.structPath) - - // Convert to DynPath and verify result - result := ConvertPathNodeToDynPath(pathNode, tt.rootType) - assert.Equal(t, tt.dynPath, result, "ConvertPathNodeToDynPath conversion should match expected DynPath format") - }) - } -} diff --git a/libs/structs/structpath/path.go b/libs/structs/structpath/path.go index 4c4ae2daef..17edaee242 100644 --- a/libs/structs/structpath/path.go +++ b/libs/structs/structpath/path.go @@ -72,6 +72,17 @@ func (p *PathNode) Field() (string, bool) { return "", false } +// StringKey returns either Field() or MapKey() if either is available +func (p *PathNode) StringKey() (string, bool) { + if p == nil { + return "", false + } + if p.index == tagStruct || p.index == tagMapKey { + return p.key, true + } + return "", false +} + func (p *PathNode) Parent() *PathNode { if p == nil { return nil @@ -106,7 +117,7 @@ func (p *PathNode) AsSlice() []*PathNode { // NewIndex creates a new PathNode for an array/slice index. func NewIndex(prev *PathNode, index int) *PathNode { if index < 0 { - panic("index msut be non-negative") + panic("index must be non-negative") } return &PathNode{ prev: prev, @@ -133,6 +144,16 @@ func NewStructField(prev *PathNode, fieldName string) *PathNode { } } +// NewStringKey creates either StructField or MapKey +// The fieldName should be the resolved field name (e.g., from JSON tag or Go field name). +func NewStringKey(prev *PathNode, fieldName string) *PathNode { + if isValidField(fieldName) { + return NewStructField(prev, fieldName) + } else { + return NewMapKey(prev, fieldName) + } +} + func NewDotStar(prev *PathNode) *PathNode { return &PathNode{ prev: prev, @@ -422,7 +443,24 @@ func isReservedFieldChar(ch byte) bool { return true case ']': // Bracket notation end return true + case '\'': + return true + case ' ': + return true + case '}': + return true + case '{': + return true default: return false } } + +func isValidField(s string) bool { + for ind, _ := range s { + if isReservedFieldChar(s[ind]) { + return false + } + } + return len(s) > 0 +} diff --git a/libs/structs/structpath/path_test.go b/libs/structs/structpath/path_test.go index a833e5611e..c0c13f5fe1 100644 --- a/libs/structs/structpath/path_test.go +++ b/libs/structs/structpath/path_test.go @@ -177,9 +177,9 @@ func TestPathNode(t *testing.T) { }, { name: "field with spaces", - node: NewStructField(nil, "field with spaces"), - String: "field with spaces", - Field: "field with spaces", + node: NewStringKey(nil, "field with spaces"), + String: "['field with spaces']", + MapKey: "field with spaces", }, { name: "field starting with digit", @@ -457,7 +457,7 @@ func TestParseErrors(t *testing.T) { func TestNewIndexPanic(t *testing.T) { defer func() { if r := recover(); r != nil { - assert.Contains(t, r.(string), "index msut be non-negative") + assert.Contains(t, r.(string), "index must be non-negative") } }() NewIndex(nil, -1) // Should panic diff --git a/libs/structs/structwalk/walk.go b/libs/structs/structwalk/walk.go index 4acf6d2ba0..5fce805ad3 100644 --- a/libs/structs/structwalk/walk.go +++ b/libs/structs/structwalk/walk.go @@ -125,7 +125,7 @@ func walkStruct(path *structpath.PathNode, s reflect.Value, visit VisitFunc) { if fieldName == "" { fieldName = sf.Name } - node := structpath.NewStructField(path, fieldName) + node := structpath.NewStringKey(path, fieldName) fieldVal := s.Field(i) // Skip zero values with omitempty unless field is explicitly forced. diff --git a/libs/structs/structwalk/walktype.go b/libs/structs/structwalk/walktype.go index 8d9ddcf20b..4d572e792f 100644 --- a/libs/structs/structwalk/walktype.go +++ b/libs/structs/structwalk/walktype.go @@ -91,7 +91,7 @@ func walkTypeValue(path *structpath.PathNode, typ reflect.Type, field *reflect.S return // unsupported map key type } // For maps, we walk the value type directly at the current path - walkTypeValue(structpath.NewBracketStar(path), typ.Elem(), nil, visit, visitedCount) + walkTypeValue(structpath.NewDotStar(path), typ.Elem(), nil, visit, visitedCount) default: // func, chan, interface, invalid, etc. -> ignore @@ -127,7 +127,7 @@ func walkTypeStruct(path *structpath.PathNode, st reflect.Type, visit VisitTypeF if fieldName == "" { fieldName = sf.Name } - node := structpath.NewStructField(path, fieldName) + node := structpath.NewStringKey(path, fieldName) walkTypeValue(node, sf.Type, &sf, visit, visitedCount) } } diff --git a/libs/structs/structwalk/walktype_test.go b/libs/structs/structwalk/walktype_test.go index f49496f26c..fded9f8055 100644 --- a/libs/structs/structwalk/walktype_test.go +++ b/libs/structs/structwalk/walktype_test.go @@ -6,7 +6,6 @@ import ( "testing" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/structs/dynpath" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" @@ -62,8 +61,8 @@ func TestTypes(t *testing.T) { "EmptyTagField": "", "EmptyTagFieldPtr": "", "IntField": 0, - `Map[*].X`: 0, - `MapPtr[*].X`: 0, + `Map.*.X`: 0, + `MapPtr.*.X`: 0, "Nested.X": 0, "NestedPtr.X": 0, "SliceString[*]": "", @@ -84,8 +83,8 @@ func TestTypeSelf(t *testing.T) { "SelfArrayPtr[*].valid_field": "", "SelfIndirect.X.valid_field": "", "SelfIndirectPtr.X.valid_field": "", - `SelfMapPtr[*].valid_field`: "", - `SelfMap[*].valid_field`: "", + `SelfMapPtr.*.valid_field`: "", + `SelfMap.*.valid_field`: "", "SelfReference.valid_field": "", "SelfSlicePtr[*].valid_field": "", "SelfSlice[*].valid_field": "", @@ -139,28 +138,28 @@ func TestTypeRoot(t *testing.T) { reflect.TypeOf(config.Root{}), 4000, 4300, // 4003 at the time of the update map[string]any{ - "bundle.target": "", - `variables[*].lookup.dashboard`: "", + "bundle.target": "", + `variables.*.lookup.dashboard`: "", - `resources.jobs[*].name`: "", - `resources.jobs[*].timeout_seconds`: 0, - `resources.jobs[*].max_concurrent_runs`: 0, - `resources.jobs[*].format`: jobs.Format(""), - `resources.jobs[*].description`: "", + `resources.jobs.*.name`: "", + `resources.jobs.*.timeout_seconds`: 0, + `resources.jobs.*.max_concurrent_runs`: 0, + `resources.jobs.*.format`: jobs.Format(""), + `resources.jobs.*.description`: "", // Verify nested task fields are accessible - `resources.jobs[*].tasks[*].task_key`: "", - `resources.jobs[*].tasks[*].notebook_task.notebook_path`: "", - `resources.jobs[*].tasks[*].spark_jar_task.main_class_name`: "", - `resources.jobs[*].tasks[*].for_each_task.inputs`: "", - `resources.jobs[*].tasks[*].for_each_task.task.task_key`: "", - `resources.jobs[*].tasks[*].for_each_task.task.notebook_task.notebook_path`: "", - `resources.jobs[*].tasks[*].new_cluster.node_type_id`: "", - `resources.jobs[*].tasks[*].new_cluster.num_workers`: 0, + `resources.jobs.*.tasks[*].task_key`: "", + `resources.jobs.*.tasks[*].notebook_task.notebook_path`: "", + `resources.jobs.*.tasks[*].spark_jar_task.main_class_name`: "", + `resources.jobs.*.tasks[*].for_each_task.inputs`: "", + `resources.jobs.*.tasks[*].for_each_task.task.task_key`: "", + `resources.jobs.*.tasks[*].for_each_task.task.notebook_task.notebook_path`: "", + `resources.jobs.*.tasks[*].new_cluster.node_type_id`: "", + `resources.jobs.*.tasks[*].new_cluster.num_workers`: 0, // Verify job cluster fields are accessible - `resources.jobs[*].job_clusters[*].job_cluster_key`: "", - `resources.jobs[*].job_clusters[*].new_cluster.num_workers`: 0, + `resources.jobs.*.job_clusters[*].job_cluster_key`: "", + `resources.jobs.*.job_clusters[*].new_cluster.num_workers`: 0, }, nil, ) @@ -174,7 +173,7 @@ func getReadonlyFields(t *testing.T, rootType reflect.Type) []string { } bundleTag := field.Tag.Get("bundle") if strings.Contains(bundleTag, "readonly") { - results = append(results, dynpath.ConvertPathNodeToDynPath(path, rootType)) + results = append(results, path.String()) } return true }) @@ -256,9 +255,9 @@ func TestWalkTypeVisited(t *testing.T) { "Inner.A", "Inner.B", "MapInner", - "MapInner[*]", - "MapInner[*].A", - "MapInner[*].B", + "MapInner.*", + "MapInner.*.A", + "MapInner.*.B", "SliceInner", "SliceInner[*]", "SliceInner[*].A", From 8c515ba364db6f2c8747d52be561a0a1d8da4acb Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 19 Sep 2025 15:47:13 +0200 Subject: [PATCH 07/20] pass PathNode to structaccess.Get --- .../config/mutator/log_resource_references.go | 3 +- bundle/direct/dresources/all_test.go | 9 +- bundle/direct/plan.go | 17 ++- libs/structs/structaccess/bundle_test.go | 8 -- libs/structs/structaccess/get.go | 86 +++++++------ libs/structs/structaccess/get_test.go | 61 ++++++++- libs/structs/structaccess/typecheck.go | 94 ++++++++------ libs/structs/structpath/path.go | 120 ++++++++++++++++-- libs/structs/structpath/path_test.go | 107 ++++++++++++++++ 9 files changed, 388 insertions(+), 117 deletions(-) diff --git a/bundle/config/mutator/log_resource_references.go b/bundle/config/mutator/log_resource_references.go index 546293ffde..aa7d4117a2 100644 --- a/bundle/config/mutator/log_resource_references.go +++ b/bundle/config/mutator/log_resource_references.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/structs/dynpath" "github.com/databricks/cli/libs/structs/structaccess" ) @@ -115,7 +116,7 @@ func truncate(s string, n int, suffix string) string { } func censorValue(ctx context.Context, v any, path dyn.Path) (string, error) { - v, err := structaccess.Get(v, path) + v, err := structaccess.Get(v, dynpath.ConvertDynPathToPathNode(path)) if err != nil { log.Infof(ctx, "internal error: path=%s: %s", path, err) return "err", err diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 7fee2829b7..9ffd43329a 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -8,8 +8,6 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/structs/dynpath" "github.com/databricks/cli/libs/structs/structaccess" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/structs/structwalk" @@ -156,10 +154,9 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W require.NotNil(t, remappedState) require.NoError(t, structwalk.Walk(newState, func(path *structpath.PathNode, val any, field *reflect.StructField) { - dynPath := dynpath.ConvertPathNodeToDynPath(path, reflect.TypeOf(newState)) - remoteValue, err := structaccess.Get(remappedState, dyn.MustPathFromString(dynPath)) + remoteValue, err := structaccess.Get(remappedState, path) if err != nil { - t.Errorf("Failed to read %s from remapped remote state %#v", dynPath, remappedState) + t.Errorf("Failed to read %s from remapped remote state %#v", path.String(), remappedState) } if val == nil { // t.Logf("Ignoring %s nil, remoteValue=%#v", path.String(), remoteValue) @@ -174,7 +171,7 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W // t.Logf("Testing %s v=%#v, remoteValue=%#v", path.String(), val, remoteValue) // We expect fields set explicitly to be preserved by testserver, which is true for all resources as of today. // If not true for your resource, add exception here: - assert.Equal(t, val, remoteValue, dynPath) + assert.Equal(t, val, remoteValue, path.String()) })) err = adapter.DoDelete(ctx, createdID) diff --git a/bundle/direct/plan.go b/bundle/direct/plan.go index bff9cf92d0..6b051068b7 100644 --- a/bundle/direct/plan.go +++ b/bundle/direct/plan.go @@ -9,13 +9,12 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structdiff" "github.com/databricks/databricks-sdk-go" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structaccess" + "github.com/databricks/cli/libs/structs/structpath" ) func (d *DeploymentUnit) Plan(ctx context.Context, client *databricks.WorkspaceClient, db *dstate.DeploymentState, inputConfig any, localOnly, refresh bool) (deployplan.ActionType, error) { @@ -68,12 +67,12 @@ func (d *DeploymentUnit) refreshRemoteState(ctx context.Context, id string) erro var ErrDelayed = errors.New("must be resolved after apply") func (d *DeploymentUnit) ResolveReferenceLocalOrRemote(ctx context.Context, db *dstate.DeploymentState, reference string, actionType deployplan.ActionType, config any) (any, error) { - path, ok := dynvar.PureReferenceToPath(reference) - if !ok || len(path) <= 3 || path[0:3].String() != d.ResourceKey { + path, ok := structpath.PureReferenceToPath(reference) + if !ok || path.Len() <= 3 || path.Prefix(3).String() != d.ResourceKey { return nil, fmt.Errorf("internal error: expected reference to %q, got %q", d.ResourceKey, reference) } - fieldPath := path[3:] + fieldPath := path.SkipPrefix(3) if fieldPath.String() == "id" { if actionType.KeepsID() { @@ -136,12 +135,12 @@ func (d *DeploymentUnit) ResolveReferenceLocalOrRemote(ctx context.Context, db * } func (d *DeploymentUnit) ResolveReferenceRemote(ctx context.Context, db *dstate.DeploymentState, reference string) (any, error) { - path, ok := dynvar.PureReferenceToPath(reference) - if !ok || len(path) <= 3 || path[0:3].String() != d.ResourceKey { + path, ok := structpath.PureReferenceToPath(reference) + if !ok || path.Len() <= 3 || path.Prefix(3).String() != d.ResourceKey { return nil, fmt.Errorf("internal error: expected reference to %q, got %q", d.ResourceKey, reference) } - fieldPath := path[3:] + fieldPath := path.SkipPrefix(3) // Handle "id" field separately - read from state, not remote state if fieldPath.String() == "id" { @@ -157,7 +156,7 @@ func (d *DeploymentUnit) ResolveReferenceRemote(ctx context.Context, db *dstate. } // ReadRemoteStateField reads a field from remote state with refresh if needed. -func (d *DeploymentUnit) ReadRemoteStateField(ctx context.Context, db *dstate.DeploymentState, fieldPath dyn.Path) (any, error) { +func (d *DeploymentUnit) ReadRemoteStateField(ctx context.Context, db *dstate.DeploymentState, fieldPath *structpath.PathNode) (any, error) { // We have options: // 1) Rely on the cached value; refresh if not cached. // 2) Always refresh, read the value. diff --git a/libs/structs/structaccess/bundle_test.go b/libs/structs/structaccess/bundle_test.go index 27b868e9bd..a93ff2be95 100644 --- a/libs/structs/structaccess/bundle_test.go +++ b/libs/structs/structaccess/bundle_test.go @@ -52,14 +52,6 @@ func TestGet_ConfigRoot_JobTagsAccess(t *testing.T) { require.Error(t, ValidateByString(reflect.TypeOf(root), "resources.jobs.my_job.tags.env.inner")) require.Error(t, ValidateByString(reflect.TypeOf(root), "resources.jobs.my_job.tags1")) - // Leading dot is allowed - v, err = GetByString(root, ".resources.jobs.my_job.tags.team") - require.NoError(t, err) - require.Equal(t, "platform", v) - require.NoError(t, ValidateByString(reflect.TypeOf(root), ".resources.jobs.my_job.tags.team")) - require.Error(t, ValidateByString(reflect.TypeOf(root), ".resources.jobs.my_job.tags.team.inner")) - require.Error(t, ValidateByString(reflect.TypeOf(root), ".resources.jobs.my_job.tags1")) - // Array indexing test (1) v, err = GetByString(root, "resources.jobs.my_job.tasks[0].task_key") require.NoError(t, err) diff --git a/libs/structs/structaccess/get.go b/libs/structs/structaccess/get.go index 4e298e24fe..f9d8f6f707 100644 --- a/libs/structs/structaccess/get.go +++ b/libs/structs/structaccess/get.go @@ -1,51 +1,46 @@ package structaccess import ( + "errors" "fmt" "reflect" "strconv" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/structs/structtag" ) // GetByString returns the value at the given path inside v. // This is a convenience function that parses the path string and calls Get. -// -// Path grammar (compatible with dyn path): -// - Struct field names and map keys separated by '.' (e.g., connection.id) -// - (Note, this prevents maps keys that are not id-like from being referenced, but this general problem with references today.) -// - Numeric indices in brackets for arrays/slices (e.g., items[0].name) -// - Leading '.' is allowed (e.g., .connection.id) -// -// Behavior: -// - For structs: a key matches a field by its json tag name (if present and not "-"). -// Embedded anonymous structs are searched. -// - For maps: a key indexes map[string]T (or string alias key types). -// - For slices/arrays: an index [N] selects the N-th element. -// - Wildcards ("*" or "[*]") are not supported and return an error. func GetByString(v any, path string) (any, error) { if path == "" { return v, nil } - dynPath, err := dyn.NewPathFromString(path) + pathNode, err := structpath.Parse(path) if err != nil { return nil, err } - return Get(v, dynPath) + return Get(v, pathNode) } // Get returns the value at the given path inside v. -func Get(v any, path dyn.Path) (any, error) { - if len(path) == 0 { +// - For structs: supports both .field and ['field'] notation +// - For maps: supports both ['key'] and .key notation +// - For slices/arrays: an index [N] selects the N-th element. +// - Wildcards ("*" or "[*]") are not supported and return an error. +func Get(v any, path *structpath.PathNode) (any, error) { + if path.IsRoot() { return v, nil } + // Convert path to slice for easier iteration + pathSegments := path.AsSlice() + cur := reflect.ValueOf(v) prefix := "" - for _, c := range path { + for _, node := range pathSegments { var ok bool cur, ok = deref(cur) if !ok { @@ -53,34 +48,47 @@ func Get(v any, path dyn.Path) (any, error) { return nil, fmt.Errorf("%s: cannot access nil value", prefix) } - if c.Key() != "" { - // Key access: struct field (by json tag) or map key. - newPrefix := prefix - if newPrefix == "" { - newPrefix = c.Key() - } else { - newPrefix = newPrefix + "." + c.Key() + if idx, isIndex := node.Index(); isIndex { + newPrefix := prefix + "[" + strconv.Itoa(idx) + "]" + kind := cur.Kind() + if kind != reflect.Slice && kind != reflect.Array { + return nil, fmt.Errorf("%s: cannot index %s", newPrefix, kind) } - nv, err := accessKey(cur, c.Key(), newPrefix) - if err != nil { - return nil, err + if idx < 0 || idx >= cur.Len() { + return nil, fmt.Errorf("%s: index out of range, length is %d", newPrefix, cur.Len()) } - cur = nv + cur = cur.Index(idx) prefix = newPrefix continue } - // Index access: slice/array - idx := c.Index() - newPrefix := prefix + "[" + strconv.Itoa(idx) + "]" - kind := cur.Kind() - if kind != reflect.Slice && kind != reflect.Array { - return nil, fmt.Errorf("%s: cannot index %s", newPrefix, kind) + if node.DotStar() || node.BracketStar() { + return nil, fmt.Errorf("wildcards not supported: %s", path.String()) + } + + var key string + var newPrefix string + + if field, isField := node.Field(); isField { + key = field + newPrefix = prefix + if newPrefix == "" { + newPrefix = key + } else { + newPrefix = newPrefix + "." + key + } + } else if mapKey, isMapKey := node.MapKey(); isMapKey { + key = mapKey + newPrefix = prefix + "[" + structpath.EncodeMapKey(key) + "]" + } else { + return nil, errors.New("unsupported path node type") } - if idx < 0 || idx >= cur.Len() { - return nil, fmt.Errorf("%s: index out of range, length is %d", newPrefix, cur.Len()) + + nv, err := accessKey(cur, key, newPrefix) + if err != nil { + return nil, err } - cur = cur.Index(idx) + cur = nv prefix = newPrefix } diff --git a/libs/structs/structaccess/get_test.go b/libs/structs/structaccess/get_test.go index 70dd6ed803..2d525027c5 100644 --- a/libs/structs/structaccess/get_test.go +++ b/libs/structs/structaccess/get_test.go @@ -137,11 +137,6 @@ func runCommonTests(t *testing.T, obj any) { path: "alias_map.foo", want: "bar", }, - { - name: "leading dot allowed", - path: ".connection.id", - want: "abc", - }, // Regular scalar fields - always return zero values { @@ -169,7 +164,7 @@ func runCommonTests(t *testing.T, obj any) { { name: "wildcard not supported", path: "items[*].id", - errFmt: "invalid path: items[*].id", + errFmt: "wildcards not supported: items[*].id", }, { name: "missing field", @@ -402,3 +397,57 @@ func TestGet_BundleTag_SkipsPromoted(t *testing.T) { require.EqualError(t, err, "hidden: field \"hidden\" not found in structaccess.host") require.EqualError(t, ValidateByString(reflect.TypeOf(host{}), "hidden"), "hidden: field \"hidden\" not found in structaccess.host") } + +func TestGet_FlexibleNotation(t *testing.T) { + type testStruct struct { + Field string `json:"field"` + Map map[string]string `json:"map"` + } + + obj := testStruct{ + Field: "value", + Map: map[string]string{"key": "mapvalue"}, + } + + tests := []struct { + name string + path string + want string + errFmt string + }{ + // Struct field access - both notations should work + { + name: "struct field with dot notation", + path: "field", + want: "value", + }, + { + name: "struct field with bracket notation", + path: "['field']", + want: "value", + }, + // Map key access - both notations should work + { + name: "map key with bracket notation", + path: "map['key']", + want: "mapvalue", + }, + { + name: "map key with dot notation", + path: "map.key", + want: "mapvalue", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := GetByString(obj, tt.path) + if tt.errFmt != "" { + require.EqualError(t, err, tt.errFmt) + return + } + require.NoError(t, err) + require.Equal(t, tt.want, got) + }) + } +} diff --git a/libs/structs/structaccess/typecheck.go b/libs/structs/structaccess/typecheck.go index 2582c963cd..43b5b70c29 100644 --- a/libs/structs/structaccess/typecheck.go +++ b/libs/structs/structaccess/typecheck.go @@ -1,11 +1,12 @@ package structaccess import ( + "errors" "fmt" "reflect" "strconv" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/structs/structtag" ) @@ -17,66 +18,85 @@ func ValidateByString(t reflect.Type, path string) error { return nil } - p, err := dyn.NewPathFromString(path) + pathNode, err := structpath.Parse(path) if err != nil { return err } - return Validate(t, p) + return Validate(t, pathNode) } // Validate reports whether the given path is valid for the provided type. // It returns nil if the path resolves fully, or an error indicating where resolution failed. -func Validate(t reflect.Type, path dyn.Path) error { - if len(path) == 0 { +func Validate(t reflect.Type, path *structpath.PathNode) error { + if path.IsRoot() { return nil } + // Convert path to slice for easier iteration + pathSegments := path.AsSlice() + cur := t prefix := "" - for _, c := range path { + for _, node := range pathSegments { // Always dereference pointers at the type level. for cur.Kind() == reflect.Pointer { cur = cur.Elem() } - if c.Key() != "" { - // Key access: struct field (by json tag) or map key. - newPrefix := prefix - if newPrefix == "" { - newPrefix = c.Key() - } else { - newPrefix = newPrefix + "." + c.Key() - } - - switch cur.Kind() { - case reflect.Struct: - sf, _, ok := FindStructFieldByKeyType(cur, c.Key()) - if !ok { - return fmt.Errorf("%s: field %q not found in %s", newPrefix, c.Key(), cur.String()) - } - cur = sf.Type - case reflect.Map: - kt := cur.Key() - if kt.Kind() != reflect.String { - return fmt.Errorf("%s: map key must be string, got %s", newPrefix, kt) - } - cur = cur.Elem() - default: - return fmt.Errorf("%s: cannot access key %q on %s", newPrefix, c.Key(), cur.Kind()) + // Handle different node types + if idx, isIndex := node.Index(); isIndex { + // Index access: slice/array + newPrefix := prefix + "[" + strconv.Itoa(idx) + "]" + kind := cur.Kind() + if kind != reflect.Slice && kind != reflect.Array { + return fmt.Errorf("%s: cannot index %s", newPrefix, kind) } + cur = cur.Elem() prefix = newPrefix continue } - // Index access: slice/array - idx := c.Index() - newPrefix := prefix + "[" + strconv.Itoa(idx) + "]" - kind := cur.Kind() - if kind != reflect.Slice && kind != reflect.Array { - return fmt.Errorf("%s: cannot index %s", newPrefix, kind) + // Handle wildcards + if node.DotStar() || node.BracketStar() { + return fmt.Errorf("wildcards not supported: %s", path.String()) + } + + // Handle field or map key access + var key string + var newPrefix string + + if field, isField := node.Field(); isField { + key = field + newPrefix = prefix + if newPrefix == "" { + newPrefix = key + } else { + newPrefix = newPrefix + "." + key + } + } else if mapKey, isMapKey := node.MapKey(); isMapKey { + key = mapKey + newPrefix = prefix + "['" + key + "']" + } else { + return errors.New("unsupported path node type") + } + + switch cur.Kind() { + case reflect.Struct: + sf, _, ok := FindStructFieldByKeyType(cur, key) + if !ok { + return fmt.Errorf("%s: field %q not found in %s", newPrefix, key, cur.String()) + } + cur = sf.Type + case reflect.Map: + kt := cur.Key() + if kt.Kind() != reflect.String { + return fmt.Errorf("%s: map key must be string, got %s", newPrefix, kt) + } + cur = cur.Elem() + default: + return fmt.Errorf("%s: cannot access key %q on %s", newPrefix, key, cur.Kind()) } - cur = cur.Elem() prefix = newPrefix } diff --git a/libs/structs/structpath/path.go b/libs/structs/structpath/path.go index 17edaee242..5aed93ffbf 100644 --- a/libs/structs/structpath/path.go +++ b/libs/structs/structpath/path.go @@ -5,6 +5,9 @@ import ( "fmt" "strconv" "strings" + + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/dynvar" ) const ( @@ -93,19 +96,14 @@ func (p *PathNode) Parent() *PathNode { // AsSlice returns the path as a slice of PathNodes from root to current. // Efficiently pre-allocates the exact length and fills in reverse order. func (p *PathNode) AsSlice() []*PathNode { - // First pass: count the length - length := 0 - current := p - for current != nil { - length++ - current = current.Parent() - } + // Use Len() to get the length efficiently + length := p.Len() // Allocate slice with exact capacity segments := make([]*PathNode, length) - // Second pass: fill in reverse order (from end to start) - current = p + // Fill in reverse order (from end to start) + current := p for i := length - 1; i >= 0; i-- { segments[i] = current current = current.Parent() @@ -209,8 +207,12 @@ func (p *PathNode) String() string { } // Format map key with single quotes, escaping single quotes by doubling them - escapedKey := strings.ReplaceAll(p.key, "'", "''") - return fmt.Sprintf("%s['%s']", p.prev.String(), escapedKey) + return fmt.Sprintf("%s[%s]", p.prev.String(), EncodeMapKey(p.key)) +} + +func EncodeMapKey(s string) string { + escaped := strings.ReplaceAll(s, "'", "''") + return "'" + escaped + "'" } // Parse parses a string representation of a path using a state machine. @@ -464,3 +466,99 @@ func isValidField(s string) bool { } return len(s) > 0 } + +// PureReferenceToPath returns a PathNode if s is a pure variable reference, otherwise false. +// This function is similar to dynvar.PureReferenceToPath but returns a *PathNode instead of dyn.Path. +func PureReferenceToPath(s string) (*PathNode, bool) { + ref, ok := dynvar.NewRef(dyn.V(s)) + if !ok { + return nil, false + } + + if !ref.IsPure() { + return nil, false + } + + pathNode, err := Parse(ref.References()[0]) + if err != nil { + return nil, false + } + + return pathNode, true +} + +// SkipPrefix returns a new PathNode that skips the first n components of the path. +// If n is greater than or equal to the path length, returns nil (root). +func (p *PathNode) SkipPrefix(n int) *PathNode { + if p.IsRoot() || n <= 0 { + return p + } + + length := p.Len() + if n >= length { + return nil // Return root + } + + startNode := p.Prefix(n) + + var result *PathNode + current := p + for current != startNode { + result = &PathNode{ + prev: result, + key: current.key, + index: current.index, + } + current = current.Parent() + } + + return result.Reverse() +} + +// Reverse returns a new PathNode with the order of components reversed. +func (p *PathNode) Reverse() *PathNode { + var result *PathNode + current := p + for current != nil { + next := current.prev + current.prev = result + result = current + current = next + } + return result +} + +// Len returns the number of components in the path. +func (p *PathNode) Len() int { + length := 0 + current := p + for current != nil { + length++ + current = current.Parent() + } + return length +} + +// Prefix returns the PathNode at the nth position (1-indexed from root). +// If n is greater than the path length, returns the entire path. +// If n <= 0, returns nil (root). +func (p *PathNode) Prefix(n int) *PathNode { + if p.IsRoot() || n <= 0 { + return nil // Return root + } + + // Find the path length first to handle edge cases + length := p.Len() + if n >= length { + return p // Return entire path + } + + // Traverse from root to find the nth node (1-indexed) + current := p + // Move to root first + for range length - n { + current = current.Parent() + } + + return current +} diff --git a/libs/structs/structpath/path_test.go b/libs/structs/structpath/path_test.go index c0c13f5fe1..a222ccbae8 100644 --- a/libs/structs/structpath/path_test.go +++ b/libs/structs/structpath/path_test.go @@ -463,3 +463,110 @@ func TestNewIndexPanic(t *testing.T) { NewIndex(nil, -1) // Should panic t.Error("Expected panic did not occur") } + +func TestPrefixAndSkipPrefix(t *testing.T) { + tests := []struct { + input string + n int + prefix string + skipPrefix string + }{ + { + input: "resources.jobs.my_job.tasks[0].name", + n: 0, + prefix: "", + skipPrefix: "resources.jobs.my_job.tasks[0].name", + }, + { + input: "resources.jobs.my_job.tasks[0].name", + n: 1, + prefix: "resources", + skipPrefix: "jobs.my_job.tasks[0].name", + }, + { + input: "resources.jobs.my_job.tasks[0].name", + n: 3, + prefix: "resources.jobs.my_job", + skipPrefix: "tasks[0].name", + }, + { + input: "resources.jobs.my_job.tasks[0].name", + n: 5, + prefix: "resources.jobs.my_job.tasks[0]", + skipPrefix: "name", + }, + { + input: "resources.jobs.my_job.tasks[0].name", + n: 6, + prefix: "resources.jobs.my_job.tasks[0].name", + skipPrefix: "", + }, + { + input: "resources.jobs.my_job.tasks[0].name", + n: 10, + prefix: "resources.jobs.my_job.tasks[0].name", + skipPrefix: "", + }, + { + input: "resources.jobs.my_job.tasks[0].name", + n: -1, + prefix: "", + skipPrefix: "resources.jobs.my_job.tasks[0].name", + }, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + path, err := Parse(tt.input) + assert.NoError(t, err) + + // Test Prefix + prefixResult := path.Prefix(tt.n) + if tt.prefix == "" { + assert.Nil(t, prefixResult) + } else { + assert.NotNil(t, prefixResult) + assert.Equal(t, tt.prefix, prefixResult.String()) + } + + // Test SkipPrefix + skipResult := path.SkipPrefix(tt.n) + if tt.skipPrefix == "" { + assert.Nil(t, skipResult) + } else { + assert.NotNil(t, skipResult) + assert.Equal(t, tt.skipPrefix, skipResult.String()) + } + }) + } +} + +func TestLen(t *testing.T) { + tests := []struct { + input string + expected int + }{ + { + input: "", + expected: 0, + }, + { + input: "field", + expected: 1, + }, + { + input: "resources.jobs['my_job'].tasks[0]", + expected: 5, + }, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + var path *PathNode + var err error + path, err = Parse(tt.input) + assert.NoError(t, err) + assert.Equal(t, tt.expected, path.Len()) + }) + } +} From bd9ed6c5781e7e36205c4d4ce1e6bc4a2bf33b83 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 22 Sep 2025 14:15:12 +0200 Subject: [PATCH 08/20] wip --- acceptance/bundle/refschema/out.fields.txt | 130 +++++++++--------- .../config/mutator/log_resource_references.go | 11 +- libs/structs/structpath/path.go | 2 +- 3 files changed, 75 insertions(+), 68 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 82b843ff62..db84dc8b40 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -75,7 +75,7 @@ resources.apps.*.compute_status *apps.ComputeStatus ALL resources.apps.*.compute_status.message string ALL resources.apps.*.compute_status.state apps.ComputeState ALL resources.apps.*.config map[string]any INPUT -resources.apps.*.config[*] any INPUT +resources.apps.*.config.* any INPUT resources.apps.*.create_time string ALL resources.apps.*.creator string ALL resources.apps.*.default_source_code_path string ALL @@ -293,7 +293,7 @@ resources.jobs.*.job_clusters[*].new_cluster.cluster_log_conf.volumes *compute.V resources.jobs.*.job_clusters[*].new_cluster.cluster_log_conf.volumes.destination string INPUT STATE resources.jobs.*.job_clusters[*].new_cluster.cluster_name string INPUT STATE resources.jobs.*.job_clusters[*].new_cluster.custom_tags map[string]string INPUT STATE -resources.jobs.*.job_clusters[*].new_cluster.custom_tags[*] string INPUT STATE +resources.jobs.*.job_clusters[*].new_cluster.custom_tags.* string INPUT STATE resources.jobs.*.job_clusters[*].new_cluster.data_security_mode compute.DataSecurityMode INPUT STATE resources.jobs.*.job_clusters[*].new_cluster.docker_image *compute.DockerImage INPUT STATE resources.jobs.*.job_clusters[*].new_cluster.docker_image.basic_auth *compute.DockerBasicAuth INPUT STATE @@ -344,9 +344,9 @@ resources.jobs.*.job_clusters[*].new_cluster.remote_disk_throughput int INPUT ST resources.jobs.*.job_clusters[*].new_cluster.runtime_engine compute.RuntimeEngine INPUT STATE resources.jobs.*.job_clusters[*].new_cluster.single_user_name string INPUT STATE resources.jobs.*.job_clusters[*].new_cluster.spark_conf map[string]string INPUT STATE -resources.jobs.*.job_clusters[*].new_cluster.spark_conf[*] string INPUT STATE +resources.jobs.*.job_clusters[*].new_cluster.spark_conf.* string INPUT STATE resources.jobs.*.job_clusters[*].new_cluster.spark_env_vars map[string]string INPUT STATE -resources.jobs.*.job_clusters[*].new_cluster.spark_env_vars[*] string INPUT STATE +resources.jobs.*.job_clusters[*].new_cluster.spark_env_vars.* string INPUT STATE resources.jobs.*.job_clusters[*].new_cluster.spark_version string INPUT STATE resources.jobs.*.job_clusters[*].new_cluster.ssh_public_keys []string INPUT STATE resources.jobs.*.job_clusters[*].new_cluster.ssh_public_keys[*] string INPUT STATE @@ -480,7 +480,7 @@ resources.jobs.*.settings.job_clusters[*].new_cluster.cluster_log_conf.volumes * resources.jobs.*.settings.job_clusters[*].new_cluster.cluster_log_conf.volumes.destination string REMOTE resources.jobs.*.settings.job_clusters[*].new_cluster.cluster_name string REMOTE resources.jobs.*.settings.job_clusters[*].new_cluster.custom_tags map[string]string REMOTE -resources.jobs.*.settings.job_clusters[*].new_cluster.custom_tags[*] string REMOTE +resources.jobs.*.settings.job_clusters[*].new_cluster.custom_tags.* string REMOTE resources.jobs.*.settings.job_clusters[*].new_cluster.data_security_mode compute.DataSecurityMode REMOTE resources.jobs.*.settings.job_clusters[*].new_cluster.docker_image *compute.DockerImage REMOTE resources.jobs.*.settings.job_clusters[*].new_cluster.docker_image.basic_auth *compute.DockerBasicAuth REMOTE @@ -531,9 +531,9 @@ resources.jobs.*.settings.job_clusters[*].new_cluster.remote_disk_throughput int resources.jobs.*.settings.job_clusters[*].new_cluster.runtime_engine compute.RuntimeEngine REMOTE resources.jobs.*.settings.job_clusters[*].new_cluster.single_user_name string REMOTE resources.jobs.*.settings.job_clusters[*].new_cluster.spark_conf map[string]string REMOTE -resources.jobs.*.settings.job_clusters[*].new_cluster.spark_conf[*] string REMOTE +resources.jobs.*.settings.job_clusters[*].new_cluster.spark_conf.* string REMOTE resources.jobs.*.settings.job_clusters[*].new_cluster.spark_env_vars map[string]string REMOTE -resources.jobs.*.settings.job_clusters[*].new_cluster.spark_env_vars[*] string REMOTE +resources.jobs.*.settings.job_clusters[*].new_cluster.spark_env_vars.* string REMOTE resources.jobs.*.settings.job_clusters[*].new_cluster.spark_version string REMOTE resources.jobs.*.settings.job_clusters[*].new_cluster.ssh_public_keys []string REMOTE resources.jobs.*.settings.job_clusters[*].new_cluster.ssh_public_keys[*] string REMOTE @@ -563,14 +563,14 @@ resources.jobs.*.settings.schedule.pause_status jobs.PauseStatus REMOTE resources.jobs.*.settings.schedule.quartz_cron_expression string REMOTE resources.jobs.*.settings.schedule.timezone_id string REMOTE resources.jobs.*.settings.tags map[string]string REMOTE -resources.jobs.*.settings.tags[*] string REMOTE +resources.jobs.*.settings.tags.* string REMOTE resources.jobs.*.settings.tasks []jobs.Task REMOTE resources.jobs.*.settings.tasks[*] jobs.Task REMOTE resources.jobs.*.settings.tasks[*].clean_rooms_notebook_task *jobs.CleanRoomsNotebookTask REMOTE resources.jobs.*.settings.tasks[*].clean_rooms_notebook_task.clean_room_name string REMOTE resources.jobs.*.settings.tasks[*].clean_rooms_notebook_task.etag string REMOTE resources.jobs.*.settings.tasks[*].clean_rooms_notebook_task.notebook_base_parameters map[string]string REMOTE -resources.jobs.*.settings.tasks[*].clean_rooms_notebook_task.notebook_base_parameters[*] string REMOTE +resources.jobs.*.settings.tasks[*].clean_rooms_notebook_task.notebook_base_parameters.* string REMOTE resources.jobs.*.settings.tasks[*].clean_rooms_notebook_task.notebook_name string REMOTE resources.jobs.*.settings.tasks[*].condition_task *jobs.ConditionTask REMOTE resources.jobs.*.settings.tasks[*].condition_task.left string REMOTE @@ -629,7 +629,7 @@ resources.jobs.*.settings.tasks[*].for_each_task.task.clean_rooms_notebook_task resources.jobs.*.settings.tasks[*].for_each_task.task.clean_rooms_notebook_task.clean_room_name string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.clean_rooms_notebook_task.etag string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.clean_rooms_notebook_task.notebook_base_parameters map[string]string REMOTE -resources.jobs.*.settings.tasks[*].for_each_task.task.clean_rooms_notebook_task.notebook_base_parameters[*] string REMOTE +resources.jobs.*.settings.tasks[*].for_each_task.task.clean_rooms_notebook_task.notebook_base_parameters.* string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.clean_rooms_notebook_task.notebook_name string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.condition_task *jobs.ConditionTask REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.condition_task.left string REMOTE @@ -761,7 +761,7 @@ resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.cluster_log_co resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.cluster_log_conf.volumes.destination string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.cluster_name string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.custom_tags map[string]string REMOTE -resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.custom_tags[*] string REMOTE +resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.custom_tags.* string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.data_security_mode compute.DataSecurityMode REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.docker_image *compute.DockerImage REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.docker_image.basic_auth *compute.DockerBasicAuth REMOTE @@ -812,9 +812,9 @@ resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.remote_disk_th resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.runtime_engine compute.RuntimeEngine REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.single_user_name string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.spark_conf map[string]string REMOTE -resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.spark_conf[*] string REMOTE +resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.spark_conf.* string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.spark_env_vars map[string]string REMOTE -resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.spark_env_vars[*] string REMOTE +resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.spark_env_vars.* string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.spark_version string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.ssh_public_keys []string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.ssh_public_keys[*] string REMOTE @@ -826,7 +826,7 @@ resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.workload_type. resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.workload_type.clients.notebooks bool REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.notebook_task *jobs.NotebookTask REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.notebook_task.base_parameters map[string]string REMOTE -resources.jobs.*.settings.tasks[*].for_each_task.task.notebook_task.base_parameters[*] string REMOTE +resources.jobs.*.settings.tasks[*].for_each_task.task.notebook_task.base_parameters.* string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.notebook_task.notebook_path string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.notebook_task.source jobs.Source REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.notebook_task.warehouse_id string REMOTE @@ -856,7 +856,7 @@ resources.jobs.*.settings.tasks[*].for_each_task.task.power_bi_task.warehouse_id resources.jobs.*.settings.tasks[*].for_each_task.task.python_wheel_task *jobs.PythonWheelTask REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.python_wheel_task.entry_point string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.python_wheel_task.named_parameters map[string]string REMOTE -resources.jobs.*.settings.tasks[*].for_each_task.task.python_wheel_task.named_parameters[*] string REMOTE +resources.jobs.*.settings.tasks[*].for_each_task.task.python_wheel_task.named_parameters.* string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.python_wheel_task.package_name string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.python_wheel_task.parameters []string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.python_wheel_task.parameters[*] string REMOTE @@ -869,19 +869,19 @@ resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.jar_params [] resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.jar_params[*] string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.job_id int64 REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.job_parameters map[string]string REMOTE -resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.job_parameters[*] string REMOTE +resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.job_parameters.* string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.notebook_params map[string]string REMOTE -resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.notebook_params[*] string REMOTE +resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.notebook_params.* string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.pipeline_params *jobs.PipelineParams REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.pipeline_params.full_refresh bool REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.python_named_params map[string]string REMOTE -resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.python_named_params[*] string REMOTE +resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.python_named_params.* string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.python_params []string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.python_params[*] string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.spark_submit_params []string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.spark_submit_params[*] string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.sql_params map[string]string REMOTE -resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.sql_params[*] string REMOTE +resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.sql_params.* string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.spark_jar_task *jobs.SparkJarTask REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.spark_jar_task.jar_uri string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.spark_jar_task.main_class_name string REMOTE @@ -916,7 +916,7 @@ resources.jobs.*.settings.tasks[*].for_each_task.task.sql_task.file *jobs.SqlTas resources.jobs.*.settings.tasks[*].for_each_task.task.sql_task.file.path string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.sql_task.file.source jobs.Source REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.sql_task.parameters map[string]string REMOTE -resources.jobs.*.settings.tasks[*].for_each_task.task.sql_task.parameters[*] string REMOTE +resources.jobs.*.settings.tasks[*].for_each_task.task.sql_task.parameters.* string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.sql_task.query *jobs.SqlTaskQuery REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.sql_task.query.query_id string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.sql_task.warehouse_id string REMOTE @@ -1015,7 +1015,7 @@ resources.jobs.*.settings.tasks[*].new_cluster.cluster_log_conf.volumes *compute resources.jobs.*.settings.tasks[*].new_cluster.cluster_log_conf.volumes.destination string REMOTE resources.jobs.*.settings.tasks[*].new_cluster.cluster_name string REMOTE resources.jobs.*.settings.tasks[*].new_cluster.custom_tags map[string]string REMOTE -resources.jobs.*.settings.tasks[*].new_cluster.custom_tags[*] string REMOTE +resources.jobs.*.settings.tasks[*].new_cluster.custom_tags.* string REMOTE resources.jobs.*.settings.tasks[*].new_cluster.data_security_mode compute.DataSecurityMode REMOTE resources.jobs.*.settings.tasks[*].new_cluster.docker_image *compute.DockerImage REMOTE resources.jobs.*.settings.tasks[*].new_cluster.docker_image.basic_auth *compute.DockerBasicAuth REMOTE @@ -1066,9 +1066,9 @@ resources.jobs.*.settings.tasks[*].new_cluster.remote_disk_throughput int REMOTE resources.jobs.*.settings.tasks[*].new_cluster.runtime_engine compute.RuntimeEngine REMOTE resources.jobs.*.settings.tasks[*].new_cluster.single_user_name string REMOTE resources.jobs.*.settings.tasks[*].new_cluster.spark_conf map[string]string REMOTE -resources.jobs.*.settings.tasks[*].new_cluster.spark_conf[*] string REMOTE +resources.jobs.*.settings.tasks[*].new_cluster.spark_conf.* string REMOTE resources.jobs.*.settings.tasks[*].new_cluster.spark_env_vars map[string]string REMOTE -resources.jobs.*.settings.tasks[*].new_cluster.spark_env_vars[*] string REMOTE +resources.jobs.*.settings.tasks[*].new_cluster.spark_env_vars.* string REMOTE resources.jobs.*.settings.tasks[*].new_cluster.spark_version string REMOTE resources.jobs.*.settings.tasks[*].new_cluster.ssh_public_keys []string REMOTE resources.jobs.*.settings.tasks[*].new_cluster.ssh_public_keys[*] string REMOTE @@ -1080,7 +1080,7 @@ resources.jobs.*.settings.tasks[*].new_cluster.workload_type.clients.jobs bool R resources.jobs.*.settings.tasks[*].new_cluster.workload_type.clients.notebooks bool REMOTE resources.jobs.*.settings.tasks[*].notebook_task *jobs.NotebookTask REMOTE resources.jobs.*.settings.tasks[*].notebook_task.base_parameters map[string]string REMOTE -resources.jobs.*.settings.tasks[*].notebook_task.base_parameters[*] string REMOTE +resources.jobs.*.settings.tasks[*].notebook_task.base_parameters.* string REMOTE resources.jobs.*.settings.tasks[*].notebook_task.notebook_path string REMOTE resources.jobs.*.settings.tasks[*].notebook_task.source jobs.Source REMOTE resources.jobs.*.settings.tasks[*].notebook_task.warehouse_id string REMOTE @@ -1110,7 +1110,7 @@ resources.jobs.*.settings.tasks[*].power_bi_task.warehouse_id string REMOTE resources.jobs.*.settings.tasks[*].python_wheel_task *jobs.PythonWheelTask REMOTE resources.jobs.*.settings.tasks[*].python_wheel_task.entry_point string REMOTE resources.jobs.*.settings.tasks[*].python_wheel_task.named_parameters map[string]string REMOTE -resources.jobs.*.settings.tasks[*].python_wheel_task.named_parameters[*] string REMOTE +resources.jobs.*.settings.tasks[*].python_wheel_task.named_parameters.* string REMOTE resources.jobs.*.settings.tasks[*].python_wheel_task.package_name string REMOTE resources.jobs.*.settings.tasks[*].python_wheel_task.parameters []string REMOTE resources.jobs.*.settings.tasks[*].python_wheel_task.parameters[*] string REMOTE @@ -1123,19 +1123,19 @@ resources.jobs.*.settings.tasks[*].run_job_task.jar_params []string REMOTE resources.jobs.*.settings.tasks[*].run_job_task.jar_params[*] string REMOTE resources.jobs.*.settings.tasks[*].run_job_task.job_id int64 REMOTE resources.jobs.*.settings.tasks[*].run_job_task.job_parameters map[string]string REMOTE -resources.jobs.*.settings.tasks[*].run_job_task.job_parameters[*] string REMOTE +resources.jobs.*.settings.tasks[*].run_job_task.job_parameters.* string REMOTE resources.jobs.*.settings.tasks[*].run_job_task.notebook_params map[string]string REMOTE -resources.jobs.*.settings.tasks[*].run_job_task.notebook_params[*] string REMOTE +resources.jobs.*.settings.tasks[*].run_job_task.notebook_params.* string REMOTE resources.jobs.*.settings.tasks[*].run_job_task.pipeline_params *jobs.PipelineParams REMOTE resources.jobs.*.settings.tasks[*].run_job_task.pipeline_params.full_refresh bool REMOTE resources.jobs.*.settings.tasks[*].run_job_task.python_named_params map[string]string REMOTE -resources.jobs.*.settings.tasks[*].run_job_task.python_named_params[*] string REMOTE +resources.jobs.*.settings.tasks[*].run_job_task.python_named_params.* string REMOTE resources.jobs.*.settings.tasks[*].run_job_task.python_params []string REMOTE resources.jobs.*.settings.tasks[*].run_job_task.python_params[*] string REMOTE resources.jobs.*.settings.tasks[*].run_job_task.spark_submit_params []string REMOTE resources.jobs.*.settings.tasks[*].run_job_task.spark_submit_params[*] string REMOTE resources.jobs.*.settings.tasks[*].run_job_task.sql_params map[string]string REMOTE -resources.jobs.*.settings.tasks[*].run_job_task.sql_params[*] string REMOTE +resources.jobs.*.settings.tasks[*].run_job_task.sql_params.* string REMOTE resources.jobs.*.settings.tasks[*].spark_jar_task *jobs.SparkJarTask REMOTE resources.jobs.*.settings.tasks[*].spark_jar_task.jar_uri string REMOTE resources.jobs.*.settings.tasks[*].spark_jar_task.main_class_name string REMOTE @@ -1170,7 +1170,7 @@ resources.jobs.*.settings.tasks[*].sql_task.file *jobs.SqlTaskFile REMOTE resources.jobs.*.settings.tasks[*].sql_task.file.path string REMOTE resources.jobs.*.settings.tasks[*].sql_task.file.source jobs.Source REMOTE resources.jobs.*.settings.tasks[*].sql_task.parameters map[string]string REMOTE -resources.jobs.*.settings.tasks[*].sql_task.parameters[*] string REMOTE +resources.jobs.*.settings.tasks[*].sql_task.parameters.* string REMOTE resources.jobs.*.settings.tasks[*].sql_task.query *jobs.SqlTaskQuery REMOTE resources.jobs.*.settings.tasks[*].sql_task.query.query_id string REMOTE resources.jobs.*.settings.tasks[*].sql_task.warehouse_id string REMOTE @@ -1232,14 +1232,14 @@ resources.jobs.*.settings.webhook_notifications.on_success []jobs.Webhook REMOTE resources.jobs.*.settings.webhook_notifications.on_success[*] jobs.Webhook REMOTE resources.jobs.*.settings.webhook_notifications.on_success[*].id string REMOTE resources.jobs.*.tags map[string]string INPUT STATE -resources.jobs.*.tags[*] string INPUT STATE +resources.jobs.*.tags.* string INPUT STATE resources.jobs.*.tasks []jobs.Task INPUT STATE resources.jobs.*.tasks[*] jobs.Task INPUT STATE resources.jobs.*.tasks[*].clean_rooms_notebook_task *jobs.CleanRoomsNotebookTask INPUT STATE resources.jobs.*.tasks[*].clean_rooms_notebook_task.clean_room_name string INPUT STATE resources.jobs.*.tasks[*].clean_rooms_notebook_task.etag string INPUT STATE resources.jobs.*.tasks[*].clean_rooms_notebook_task.notebook_base_parameters map[string]string INPUT STATE -resources.jobs.*.tasks[*].clean_rooms_notebook_task.notebook_base_parameters[*] string INPUT STATE +resources.jobs.*.tasks[*].clean_rooms_notebook_task.notebook_base_parameters.* string INPUT STATE resources.jobs.*.tasks[*].clean_rooms_notebook_task.notebook_name string INPUT STATE resources.jobs.*.tasks[*].condition_task *jobs.ConditionTask INPUT STATE resources.jobs.*.tasks[*].condition_task.left string INPUT STATE @@ -1298,7 +1298,7 @@ resources.jobs.*.tasks[*].for_each_task.task.clean_rooms_notebook_task *jobs.Cle resources.jobs.*.tasks[*].for_each_task.task.clean_rooms_notebook_task.clean_room_name string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.clean_rooms_notebook_task.etag string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.clean_rooms_notebook_task.notebook_base_parameters map[string]string INPUT STATE -resources.jobs.*.tasks[*].for_each_task.task.clean_rooms_notebook_task.notebook_base_parameters[*] string INPUT STATE +resources.jobs.*.tasks[*].for_each_task.task.clean_rooms_notebook_task.notebook_base_parameters.* string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.clean_rooms_notebook_task.notebook_name string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.condition_task *jobs.ConditionTask INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.condition_task.left string INPUT STATE @@ -1430,7 +1430,7 @@ resources.jobs.*.tasks[*].for_each_task.task.new_cluster.cluster_log_conf.volume resources.jobs.*.tasks[*].for_each_task.task.new_cluster.cluster_log_conf.volumes.destination string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.new_cluster.cluster_name string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.new_cluster.custom_tags map[string]string INPUT STATE -resources.jobs.*.tasks[*].for_each_task.task.new_cluster.custom_tags[*] string INPUT STATE +resources.jobs.*.tasks[*].for_each_task.task.new_cluster.custom_tags.* string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.new_cluster.data_security_mode compute.DataSecurityMode INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.new_cluster.docker_image *compute.DockerImage INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.new_cluster.docker_image.basic_auth *compute.DockerBasicAuth INPUT STATE @@ -1481,9 +1481,9 @@ resources.jobs.*.tasks[*].for_each_task.task.new_cluster.remote_disk_throughput resources.jobs.*.tasks[*].for_each_task.task.new_cluster.runtime_engine compute.RuntimeEngine INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.new_cluster.single_user_name string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.new_cluster.spark_conf map[string]string INPUT STATE -resources.jobs.*.tasks[*].for_each_task.task.new_cluster.spark_conf[*] string INPUT STATE +resources.jobs.*.tasks[*].for_each_task.task.new_cluster.spark_conf.* string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.new_cluster.spark_env_vars map[string]string INPUT STATE -resources.jobs.*.tasks[*].for_each_task.task.new_cluster.spark_env_vars[*] string INPUT STATE +resources.jobs.*.tasks[*].for_each_task.task.new_cluster.spark_env_vars.* string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.new_cluster.spark_version string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.new_cluster.ssh_public_keys []string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.new_cluster.ssh_public_keys[*] string INPUT STATE @@ -1495,7 +1495,7 @@ resources.jobs.*.tasks[*].for_each_task.task.new_cluster.workload_type.clients.j resources.jobs.*.tasks[*].for_each_task.task.new_cluster.workload_type.clients.notebooks bool INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.notebook_task *jobs.NotebookTask INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.notebook_task.base_parameters map[string]string INPUT STATE -resources.jobs.*.tasks[*].for_each_task.task.notebook_task.base_parameters[*] string INPUT STATE +resources.jobs.*.tasks[*].for_each_task.task.notebook_task.base_parameters.* string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.notebook_task.notebook_path string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.notebook_task.source jobs.Source INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.notebook_task.warehouse_id string INPUT STATE @@ -1525,7 +1525,7 @@ resources.jobs.*.tasks[*].for_each_task.task.power_bi_task.warehouse_id string I resources.jobs.*.tasks[*].for_each_task.task.python_wheel_task *jobs.PythonWheelTask INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.python_wheel_task.entry_point string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.python_wheel_task.named_parameters map[string]string INPUT STATE -resources.jobs.*.tasks[*].for_each_task.task.python_wheel_task.named_parameters[*] string INPUT STATE +resources.jobs.*.tasks[*].for_each_task.task.python_wheel_task.named_parameters.* string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.python_wheel_task.package_name string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.python_wheel_task.parameters []string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.python_wheel_task.parameters[*] string INPUT STATE @@ -1538,19 +1538,19 @@ resources.jobs.*.tasks[*].for_each_task.task.run_job_task.jar_params []string IN resources.jobs.*.tasks[*].for_each_task.task.run_job_task.jar_params[*] string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.run_job_task.job_id int64 INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.run_job_task.job_parameters map[string]string INPUT STATE -resources.jobs.*.tasks[*].for_each_task.task.run_job_task.job_parameters[*] string INPUT STATE +resources.jobs.*.tasks[*].for_each_task.task.run_job_task.job_parameters.* string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.run_job_task.notebook_params map[string]string INPUT STATE -resources.jobs.*.tasks[*].for_each_task.task.run_job_task.notebook_params[*] string INPUT STATE +resources.jobs.*.tasks[*].for_each_task.task.run_job_task.notebook_params.* string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.run_job_task.pipeline_params *jobs.PipelineParams INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.run_job_task.pipeline_params.full_refresh bool INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.run_job_task.python_named_params map[string]string INPUT STATE -resources.jobs.*.tasks[*].for_each_task.task.run_job_task.python_named_params[*] string INPUT STATE +resources.jobs.*.tasks[*].for_each_task.task.run_job_task.python_named_params.* string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.run_job_task.python_params []string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.run_job_task.python_params[*] string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.run_job_task.spark_submit_params []string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.run_job_task.spark_submit_params[*] string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.run_job_task.sql_params map[string]string INPUT STATE -resources.jobs.*.tasks[*].for_each_task.task.run_job_task.sql_params[*] string INPUT STATE +resources.jobs.*.tasks[*].for_each_task.task.run_job_task.sql_params.* string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.spark_jar_task *jobs.SparkJarTask INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.spark_jar_task.jar_uri string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.spark_jar_task.main_class_name string INPUT STATE @@ -1585,7 +1585,7 @@ resources.jobs.*.tasks[*].for_each_task.task.sql_task.file *jobs.SqlTaskFile INP resources.jobs.*.tasks[*].for_each_task.task.sql_task.file.path string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.sql_task.file.source jobs.Source INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.sql_task.parameters map[string]string INPUT STATE -resources.jobs.*.tasks[*].for_each_task.task.sql_task.parameters[*] string INPUT STATE +resources.jobs.*.tasks[*].for_each_task.task.sql_task.parameters.* string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.sql_task.query *jobs.SqlTaskQuery INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.sql_task.query.query_id string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.sql_task.warehouse_id string INPUT STATE @@ -1684,7 +1684,7 @@ resources.jobs.*.tasks[*].new_cluster.cluster_log_conf.volumes *compute.VolumesS resources.jobs.*.tasks[*].new_cluster.cluster_log_conf.volumes.destination string INPUT STATE resources.jobs.*.tasks[*].new_cluster.cluster_name string INPUT STATE resources.jobs.*.tasks[*].new_cluster.custom_tags map[string]string INPUT STATE -resources.jobs.*.tasks[*].new_cluster.custom_tags[*] string INPUT STATE +resources.jobs.*.tasks[*].new_cluster.custom_tags.* string INPUT STATE resources.jobs.*.tasks[*].new_cluster.data_security_mode compute.DataSecurityMode INPUT STATE resources.jobs.*.tasks[*].new_cluster.docker_image *compute.DockerImage INPUT STATE resources.jobs.*.tasks[*].new_cluster.docker_image.basic_auth *compute.DockerBasicAuth INPUT STATE @@ -1735,9 +1735,9 @@ resources.jobs.*.tasks[*].new_cluster.remote_disk_throughput int INPUT STATE resources.jobs.*.tasks[*].new_cluster.runtime_engine compute.RuntimeEngine INPUT STATE resources.jobs.*.tasks[*].new_cluster.single_user_name string INPUT STATE resources.jobs.*.tasks[*].new_cluster.spark_conf map[string]string INPUT STATE -resources.jobs.*.tasks[*].new_cluster.spark_conf[*] string INPUT STATE +resources.jobs.*.tasks[*].new_cluster.spark_conf.* string INPUT STATE resources.jobs.*.tasks[*].new_cluster.spark_env_vars map[string]string INPUT STATE -resources.jobs.*.tasks[*].new_cluster.spark_env_vars[*] string INPUT STATE +resources.jobs.*.tasks[*].new_cluster.spark_env_vars.* string INPUT STATE resources.jobs.*.tasks[*].new_cluster.spark_version string INPUT STATE resources.jobs.*.tasks[*].new_cluster.ssh_public_keys []string INPUT STATE resources.jobs.*.tasks[*].new_cluster.ssh_public_keys[*] string INPUT STATE @@ -1749,7 +1749,7 @@ resources.jobs.*.tasks[*].new_cluster.workload_type.clients.jobs bool INPUT STAT resources.jobs.*.tasks[*].new_cluster.workload_type.clients.notebooks bool INPUT STATE resources.jobs.*.tasks[*].notebook_task *jobs.NotebookTask INPUT STATE resources.jobs.*.tasks[*].notebook_task.base_parameters map[string]string INPUT STATE -resources.jobs.*.tasks[*].notebook_task.base_parameters[*] string INPUT STATE +resources.jobs.*.tasks[*].notebook_task.base_parameters.* string INPUT STATE resources.jobs.*.tasks[*].notebook_task.notebook_path string INPUT STATE resources.jobs.*.tasks[*].notebook_task.source jobs.Source INPUT STATE resources.jobs.*.tasks[*].notebook_task.warehouse_id string INPUT STATE @@ -1779,7 +1779,7 @@ resources.jobs.*.tasks[*].power_bi_task.warehouse_id string INPUT STATE resources.jobs.*.tasks[*].python_wheel_task *jobs.PythonWheelTask INPUT STATE resources.jobs.*.tasks[*].python_wheel_task.entry_point string INPUT STATE resources.jobs.*.tasks[*].python_wheel_task.named_parameters map[string]string INPUT STATE -resources.jobs.*.tasks[*].python_wheel_task.named_parameters[*] string INPUT STATE +resources.jobs.*.tasks[*].python_wheel_task.named_parameters.* string INPUT STATE resources.jobs.*.tasks[*].python_wheel_task.package_name string INPUT STATE resources.jobs.*.tasks[*].python_wheel_task.parameters []string INPUT STATE resources.jobs.*.tasks[*].python_wheel_task.parameters[*] string INPUT STATE @@ -1792,19 +1792,19 @@ resources.jobs.*.tasks[*].run_job_task.jar_params []string INPUT STATE resources.jobs.*.tasks[*].run_job_task.jar_params[*] string INPUT STATE resources.jobs.*.tasks[*].run_job_task.job_id int64 INPUT STATE resources.jobs.*.tasks[*].run_job_task.job_parameters map[string]string INPUT STATE -resources.jobs.*.tasks[*].run_job_task.job_parameters[*] string INPUT STATE +resources.jobs.*.tasks[*].run_job_task.job_parameters.* string INPUT STATE resources.jobs.*.tasks[*].run_job_task.notebook_params map[string]string INPUT STATE -resources.jobs.*.tasks[*].run_job_task.notebook_params[*] string INPUT STATE +resources.jobs.*.tasks[*].run_job_task.notebook_params.* string INPUT STATE resources.jobs.*.tasks[*].run_job_task.pipeline_params *jobs.PipelineParams INPUT STATE resources.jobs.*.tasks[*].run_job_task.pipeline_params.full_refresh bool INPUT STATE resources.jobs.*.tasks[*].run_job_task.python_named_params map[string]string INPUT STATE -resources.jobs.*.tasks[*].run_job_task.python_named_params[*] string INPUT STATE +resources.jobs.*.tasks[*].run_job_task.python_named_params.* string INPUT STATE resources.jobs.*.tasks[*].run_job_task.python_params []string INPUT STATE resources.jobs.*.tasks[*].run_job_task.python_params[*] string INPUT STATE resources.jobs.*.tasks[*].run_job_task.spark_submit_params []string INPUT STATE resources.jobs.*.tasks[*].run_job_task.spark_submit_params[*] string INPUT STATE resources.jobs.*.tasks[*].run_job_task.sql_params map[string]string INPUT STATE -resources.jobs.*.tasks[*].run_job_task.sql_params[*] string INPUT STATE +resources.jobs.*.tasks[*].run_job_task.sql_params.* string INPUT STATE resources.jobs.*.tasks[*].spark_jar_task *jobs.SparkJarTask INPUT STATE resources.jobs.*.tasks[*].spark_jar_task.jar_uri string INPUT STATE resources.jobs.*.tasks[*].spark_jar_task.main_class_name string INPUT STATE @@ -1839,7 +1839,7 @@ resources.jobs.*.tasks[*].sql_task.file *jobs.SqlTaskFile INPUT STATE resources.jobs.*.tasks[*].sql_task.file.path string INPUT STATE resources.jobs.*.tasks[*].sql_task.file.source jobs.Source INPUT STATE resources.jobs.*.tasks[*].sql_task.parameters map[string]string INPUT STATE -resources.jobs.*.tasks[*].sql_task.parameters[*] string INPUT STATE +resources.jobs.*.tasks[*].sql_task.parameters.* string INPUT STATE resources.jobs.*.tasks[*].sql_task.query *jobs.SqlTaskQuery INPUT STATE resources.jobs.*.tasks[*].sql_task.query.query_id string INPUT STATE resources.jobs.*.tasks[*].sql_task.warehouse_id string INPUT STATE @@ -1955,7 +1955,7 @@ resources.pipelines.*.clusters[*].cluster_log_conf.s3.region string INPUT STATE resources.pipelines.*.clusters[*].cluster_log_conf.volumes *compute.VolumesStorageInfo INPUT STATE resources.pipelines.*.clusters[*].cluster_log_conf.volumes.destination string INPUT STATE resources.pipelines.*.clusters[*].custom_tags map[string]string INPUT STATE -resources.pipelines.*.clusters[*].custom_tags[*] string INPUT STATE +resources.pipelines.*.clusters[*].custom_tags.* string INPUT STATE resources.pipelines.*.clusters[*].driver_instance_pool_id string INPUT STATE resources.pipelines.*.clusters[*].driver_node_type_id string INPUT STATE resources.pipelines.*.clusters[*].enable_local_disk_encryption bool INPUT STATE @@ -1995,13 +1995,13 @@ resources.pipelines.*.clusters[*].node_type_id string INPUT STATE resources.pipelines.*.clusters[*].num_workers int INPUT STATE resources.pipelines.*.clusters[*].policy_id string INPUT STATE resources.pipelines.*.clusters[*].spark_conf map[string]string INPUT STATE -resources.pipelines.*.clusters[*].spark_conf[*] string INPUT STATE +resources.pipelines.*.clusters[*].spark_conf.* string INPUT STATE resources.pipelines.*.clusters[*].spark_env_vars map[string]string INPUT STATE -resources.pipelines.*.clusters[*].spark_env_vars[*] string INPUT STATE +resources.pipelines.*.clusters[*].spark_env_vars.* string INPUT STATE resources.pipelines.*.clusters[*].ssh_public_keys []string INPUT STATE resources.pipelines.*.clusters[*].ssh_public_keys[*] string INPUT STATE resources.pipelines.*.configuration map[string]string INPUT STATE -resources.pipelines.*.configuration[*] string INPUT STATE +resources.pipelines.*.configuration.* string INPUT STATE resources.pipelines.*.continuous bool INPUT STATE resources.pipelines.*.creator_user_name string REMOTE resources.pipelines.*.deployment *pipelines.PipelineDeployment INPUT STATE @@ -2220,7 +2220,7 @@ resources.pipelines.*.spec.clusters[*].cluster_log_conf.s3.region string REMOTE resources.pipelines.*.spec.clusters[*].cluster_log_conf.volumes *compute.VolumesStorageInfo REMOTE resources.pipelines.*.spec.clusters[*].cluster_log_conf.volumes.destination string REMOTE resources.pipelines.*.spec.clusters[*].custom_tags map[string]string REMOTE -resources.pipelines.*.spec.clusters[*].custom_tags[*] string REMOTE +resources.pipelines.*.spec.clusters[*].custom_tags.* string REMOTE resources.pipelines.*.spec.clusters[*].driver_instance_pool_id string REMOTE resources.pipelines.*.spec.clusters[*].driver_node_type_id string REMOTE resources.pipelines.*.spec.clusters[*].enable_local_disk_encryption bool REMOTE @@ -2260,13 +2260,13 @@ resources.pipelines.*.spec.clusters[*].node_type_id string REMOTE resources.pipelines.*.spec.clusters[*].num_workers int REMOTE resources.pipelines.*.spec.clusters[*].policy_id string REMOTE resources.pipelines.*.spec.clusters[*].spark_conf map[string]string REMOTE -resources.pipelines.*.spec.clusters[*].spark_conf[*] string REMOTE +resources.pipelines.*.spec.clusters[*].spark_conf.* string REMOTE resources.pipelines.*.spec.clusters[*].spark_env_vars map[string]string REMOTE -resources.pipelines.*.spec.clusters[*].spark_env_vars[*] string REMOTE +resources.pipelines.*.spec.clusters[*].spark_env_vars.* string REMOTE resources.pipelines.*.spec.clusters[*].ssh_public_keys []string REMOTE resources.pipelines.*.spec.clusters[*].ssh_public_keys[*] string REMOTE resources.pipelines.*.spec.configuration map[string]string REMOTE -resources.pipelines.*.spec.configuration[*] string REMOTE +resources.pipelines.*.spec.configuration.* string REMOTE resources.pipelines.*.spec.continuous bool REMOTE resources.pipelines.*.spec.deployment *pipelines.PipelineDeployment REMOTE resources.pipelines.*.spec.deployment.kind pipelines.DeploymentKind REMOTE @@ -2420,7 +2420,7 @@ resources.pipelines.*.spec.schema string REMOTE resources.pipelines.*.spec.serverless bool REMOTE resources.pipelines.*.spec.storage string REMOTE resources.pipelines.*.spec.tags map[string]string REMOTE -resources.pipelines.*.spec.tags[*] string REMOTE +resources.pipelines.*.spec.tags.* string REMOTE resources.pipelines.*.spec.target string REMOTE resources.pipelines.*.spec.trigger *pipelines.PipelineTrigger REMOTE resources.pipelines.*.spec.trigger.cron *pipelines.CronTrigger REMOTE @@ -2430,7 +2430,7 @@ resources.pipelines.*.spec.trigger.manual *pipelines.ManualTrigger REMOTE resources.pipelines.*.state pipelines.PipelineState REMOTE resources.pipelines.*.storage string INPUT STATE resources.pipelines.*.tags map[string]string INPUT STATE -resources.pipelines.*.tags[*] string INPUT STATE +resources.pipelines.*.tags.* string INPUT STATE resources.pipelines.*.target string INPUT STATE resources.pipelines.*.trigger *pipelines.PipelineTrigger INPUT STATE resources.pipelines.*.trigger.cron *pipelines.CronTrigger INPUT STATE @@ -2463,7 +2463,7 @@ resources.schemas.*.modified_status string INPUT resources.schemas.*.name string ALL resources.schemas.*.owner string REMOTE resources.schemas.*.properties map[string]string ALL -resources.schemas.*.properties[*] string ALL +resources.schemas.*.properties.* string ALL resources.schemas.*.schema_id string REMOTE resources.schemas.*.storage_location string REMOTE resources.schemas.*.storage_root string ALL @@ -2483,7 +2483,7 @@ resources.sql_warehouses.*.health.details string REMOTE resources.sql_warehouses.*.health.failure_reason *sql.TerminationReason REMOTE resources.sql_warehouses.*.health.failure_reason.code sql.TerminationReasonCode REMOTE resources.sql_warehouses.*.health.failure_reason.parameters map[string]string REMOTE -resources.sql_warehouses.*.health.failure_reason.parameters[*] string REMOTE +resources.sql_warehouses.*.health.failure_reason.parameters.* string REMOTE resources.sql_warehouses.*.health.failure_reason.type sql.TerminationReasonType REMOTE resources.sql_warehouses.*.health.message string REMOTE resources.sql_warehouses.*.health.status sql.Status REMOTE diff --git a/bundle/config/mutator/log_resource_references.go b/bundle/config/mutator/log_resource_references.go index aa7d4117a2..14c6beed49 100644 --- a/bundle/config/mutator/log_resource_references.go +++ b/bundle/config/mutator/log_resource_references.go @@ -10,8 +10,8 @@ import ( "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" - "github.com/databricks/cli/libs/structs/dynpath" "github.com/databricks/cli/libs/structs/structaccess" + "github.com/databricks/cli/libs/structs/structpath" ) // Longest field name: @@ -116,7 +116,14 @@ func truncate(s string, n int, suffix string) string { } func censorValue(ctx context.Context, v any, path dyn.Path) (string, error) { - v, err := structaccess.Get(v, dynpath.ConvertDynPathToPathNode(path)) + pathString := path.String() + pathNode, err := structpath.Parse(pathString) + if err != nil { + log.Warnf(ctx, "internal error: parsing %q: %s", pathString, err) + return "err", err + } + + v, err = structaccess.Get(v, pathNode) if err != nil { log.Infof(ctx, "internal error: path=%s: %s", path, err) return "err", err diff --git a/libs/structs/structpath/path.go b/libs/structs/structpath/path.go index 5aed93ffbf..cc5c62896e 100644 --- a/libs/structs/structpath/path.go +++ b/libs/structs/structpath/path.go @@ -459,7 +459,7 @@ func isReservedFieldChar(ch byte) bool { } func isValidField(s string) bool { - for ind, _ := range s { + for ind := range s { if isReservedFieldChar(s[ind]) { return false } From fa5ddffe21569bc6824626195ad888de74452435 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 22 Sep 2025 14:19:35 +0200 Subject: [PATCH 09/20] update structdiff tests --- libs/structs/structdiff/diff.go | 4 ++-- libs/structs/structdiff/diff_test.go | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/libs/structs/structdiff/diff.go b/libs/structs/structdiff/diff.go index e86e323b84..91be5a05c4 100644 --- a/libs/structs/structdiff/diff.go +++ b/libs/structs/structdiff/diff.go @@ -134,7 +134,7 @@ func diffStruct(path *structpath.PathNode, s1, s2 reflect.Value, changes *[]Chan if fieldName == "" { fieldName = sf.Name } - node := structpath.NewStructField(path, fieldName) + node := structpath.NewStringKey(path, fieldName) v1Field := s1.Field(i) v2Field := s2.Field(i) @@ -183,7 +183,7 @@ func diffMapStringKey(path *structpath.PathNode, m1, m2 reflect.Value, changes * k := keySet[ks] v1 := m1.MapIndex(k) v2 := m2.MapIndex(k) - node := structpath.NewMapKey(path, ks) + node := structpath.NewStringKey(path, ks) diffValues(node, v1, v2, changes) } } diff --git a/libs/structs/structdiff/diff_test.go b/libs/structs/structdiff/diff_test.go index 794e0954ff..3c69eefdcd 100644 --- a/libs/structs/structdiff/diff_test.go +++ b/libs/structs/structdiff/diff_test.go @@ -132,7 +132,7 @@ func TestGetStructDiff(t *testing.T) { name: "map diff", a: A{M: map[string]int{"a": 1}}, b: A{M: map[string]int{"a": 2}}, - want: []ResolvedChange{{Field: "m['a']", Old: 1, New: 2}}, + want: []ResolvedChange{{Field: "m.a", Old: 1, New: 2}}, }, { name: "slice diff", @@ -243,9 +243,9 @@ func TestGetStructDiff(t *testing.T) { a: map[string]C{"key1": {Title: "title", ForceSendFields: []string{"Name", "IsEnabled", "Title"}}}, b: map[string]C{"key1": {Title: "title", ForceSendFields: []string{"Age"}}}, want: []ResolvedChange{ - {Field: "['key1'].name", Old: "", New: nil}, - {Field: "['key1'].age", Old: nil, New: 0}, - {Field: "['key1'].is_enabled", Old: false, New: nil}, + {Field: "key1.name", Old: "", New: nil}, + {Field: "key1.age", Old: nil, New: 0}, + {Field: "key1.is_enabled", Old: false, New: nil}, }, }, @@ -255,9 +255,9 @@ func TestGetStructDiff(t *testing.T) { a: map[string]*C{"key1": {Title: "title", ForceSendFields: []string{"Name", "IsEnabled", "Title"}}}, b: map[string]*C{"key1": {Title: "title", ForceSendFields: []string{"Age"}}}, want: []ResolvedChange{ - {Field: "['key1'].name", Old: "", New: nil}, - {Field: "['key1'].age", Old: nil, New: 0}, - {Field: "['key1'].is_enabled", Old: false, New: nil}, + {Field: "key1.name", Old: "", New: nil}, + {Field: "key1.age", Old: nil, New: 0}, + {Field: "key1.is_enabled", Old: false, New: nil}, }, }, } From 2059ea8eff8bebc535d65a63946fa5a46ffdb737 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 22 Sep 2025 14:49:13 +0200 Subject: [PATCH 10/20] update --- bundle/internal/validation/required.go | 2 +- libs/structs/structaccess/get.go | 40 ++--- libs/structs/structaccess/typecheck.go | 34 +--- libs/structs/structpath/path.go | 66 ++------ libs/structs/structpath/path_test.go | 210 +++++++++---------------- libs/structs/structwalk/walk.go | 2 +- libs/structs/structwalk/walk_test.go | 4 +- 7 files changed, 110 insertions(+), 248 deletions(-) diff --git a/bundle/internal/validation/required.go b/bundle/internal/validation/required.go index d40e149d65..e0fd6270c3 100644 --- a/bundle/internal/validation/required.go +++ b/bundle/internal/validation/required.go @@ -63,7 +63,7 @@ func extractRequiredFields(typ reflect.Type) ([]RequiredPatternInfo, error) { return true } - fieldName, ok := path.Field() + fieldName, ok := path.StringKey() if !ok { return true } diff --git a/libs/structs/structaccess/get.go b/libs/structs/structaccess/get.go index f9d8f6f707..4817907567 100644 --- a/libs/structs/structaccess/get.go +++ b/libs/structs/structaccess/get.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" "reflect" - "strconv" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/structs/structtag" @@ -39,26 +38,23 @@ func Get(v any, path *structpath.PathNode) (any, error) { pathSegments := path.AsSlice() cur := reflect.ValueOf(v) - prefix := "" for _, node := range pathSegments { var ok bool cur, ok = deref(cur) if !ok { // cannot proceed further due to nil encountered at current location - return nil, fmt.Errorf("%s: cannot access nil value", prefix) + return nil, fmt.Errorf("%s: cannot access nil value", node.Parent().String()) } if idx, isIndex := node.Index(); isIndex { - newPrefix := prefix + "[" + strconv.Itoa(idx) + "]" kind := cur.Kind() if kind != reflect.Slice && kind != reflect.Array { - return nil, fmt.Errorf("%s: cannot index %s", newPrefix, kind) + return nil, fmt.Errorf("%s: cannot index %s", node.String(), kind) } if idx < 0 || idx >= cur.Len() { - return nil, fmt.Errorf("%s: index out of range, length is %d", newPrefix, cur.Len()) + return nil, fmt.Errorf("%s: index out of range, length is %d", node.String(), cur.Len()) } cur = cur.Index(idx) - prefix = newPrefix continue } @@ -66,30 +62,16 @@ func Get(v any, path *structpath.PathNode) (any, error) { return nil, fmt.Errorf("wildcards not supported: %s", path.String()) } - var key string - var newPrefix string - - if field, isField := node.Field(); isField { - key = field - newPrefix = prefix - if newPrefix == "" { - newPrefix = key - } else { - newPrefix = newPrefix + "." + key - } - } else if mapKey, isMapKey := node.MapKey(); isMapKey { - key = mapKey - newPrefix = prefix + "[" + structpath.EncodeMapKey(key) + "]" - } else { + key, ok := node.StringKey() + if !ok { return nil, errors.New("unsupported path node type") } - nv, err := accessKey(cur, key, newPrefix) + nv, err := accessKey(cur, key, node) if err != nil { return nil, err } cur = nv - prefix = newPrefix } // If the current value is invalid (e.g., omitted due to omitempty), return nil. @@ -108,12 +90,12 @@ func Get(v any, path *structpath.PathNode) (any, error) { // accessKey returns the field or map entry value selected by key from v. // v must be non-pointer, non-interface reflect.Value. -func accessKey(v reflect.Value, key, prefix string) (reflect.Value, error) { +func accessKey(v reflect.Value, key string, path *structpath.PathNode) (reflect.Value, error) { switch v.Kind() { case reflect.Struct: fv, sf, owner, ok := findStructFieldByKey(v, key) if !ok { - return reflect.Value{}, fmt.Errorf("%s: field %q not found in %s", prefix, key, v.Type()) + return reflect.Value{}, fmt.Errorf("%s: field %q not found in %s", path.String(), key, v.Type()) } // Evaluate ForceSendFields on both the current struct and the declaring owner force := containsForceSendField(v, sf.Name) || containsForceSendField(owner, sf.Name) @@ -137,7 +119,7 @@ func accessKey(v reflect.Value, key, prefix string) (reflect.Value, error) { case reflect.Map: kt := v.Type().Key() if kt.Kind() != reflect.String { - return reflect.Value{}, fmt.Errorf("%s: map key must be string, got %s", prefix, kt) + return reflect.Value{}, fmt.Errorf("%s: map key must be string, got %s", path.String(), kt) } mk := reflect.ValueOf(key) if kt != mk.Type() { @@ -145,11 +127,11 @@ func accessKey(v reflect.Value, key, prefix string) (reflect.Value, error) { } mv := v.MapIndex(mk) if !mv.IsValid() { - return reflect.Value{}, fmt.Errorf("%s: key %q not found in map", prefix, key) + return reflect.Value{}, fmt.Errorf("%s: key %q not found in map", path.String(), key) } return mv, nil default: - return reflect.Value{}, fmt.Errorf("%s: cannot access key %q on %s", prefix, key, v.Kind()) + return reflect.Value{}, fmt.Errorf("%s: cannot access key %q on %s", path.String(), key, v.Kind()) } } diff --git a/libs/structs/structaccess/typecheck.go b/libs/structs/structaccess/typecheck.go index 43b5b70c29..292d98ec89 100644 --- a/libs/structs/structaccess/typecheck.go +++ b/libs/structs/structaccess/typecheck.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" "reflect" - "strconv" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/structs/structtag" @@ -37,7 +36,6 @@ func Validate(t reflect.Type, path *structpath.PathNode) error { pathSegments := path.AsSlice() cur := t - prefix := "" for _, node := range pathSegments { // Always dereference pointers at the type level. for cur.Kind() == reflect.Pointer { @@ -45,15 +43,13 @@ func Validate(t reflect.Type, path *structpath.PathNode) error { } // Handle different node types - if idx, isIndex := node.Index(); isIndex { + if _, isIndex := node.Index(); isIndex { // Index access: slice/array - newPrefix := prefix + "[" + strconv.Itoa(idx) + "]" kind := cur.Kind() if kind != reflect.Slice && kind != reflect.Array { - return fmt.Errorf("%s: cannot index %s", newPrefix, kind) + return fmt.Errorf("%s: cannot index %s", node.String(), kind) } cur = cur.Elem() - prefix = newPrefix continue } @@ -62,22 +58,9 @@ func Validate(t reflect.Type, path *structpath.PathNode) error { return fmt.Errorf("wildcards not supported: %s", path.String()) } - // Handle field or map key access - var key string - var newPrefix string - - if field, isField := node.Field(); isField { - key = field - newPrefix = prefix - if newPrefix == "" { - newPrefix = key - } else { - newPrefix = newPrefix + "." + key - } - } else if mapKey, isMapKey := node.MapKey(); isMapKey { - key = mapKey - newPrefix = prefix + "['" + key + "']" - } else { + key, ok := node.StringKey() + + if !ok { return errors.New("unsupported path node type") } @@ -85,19 +68,18 @@ func Validate(t reflect.Type, path *structpath.PathNode) error { case reflect.Struct: sf, _, ok := FindStructFieldByKeyType(cur, key) if !ok { - return fmt.Errorf("%s: field %q not found in %s", newPrefix, key, cur.String()) + return fmt.Errorf("%s: field %q not found in %s", node.String(), key, cur.String()) } cur = sf.Type case reflect.Map: kt := cur.Key() if kt.Kind() != reflect.String { - return fmt.Errorf("%s: map key must be string, got %s", newPrefix, kt) + return fmt.Errorf("%s: map key must be string, got %s", node.String(), kt) } cur = cur.Elem() default: - return fmt.Errorf("%s: cannot access key %q on %s", newPrefix, key, cur.Kind()) + return fmt.Errorf("%s: cannot access key %q on %s", node.String(), key, cur.Kind()) } - prefix = newPrefix } return nil diff --git a/libs/structs/structpath/path.go b/libs/structs/structpath/path.go index cc5c62896e..22ec85fa39 100644 --- a/libs/structs/structpath/path.go +++ b/libs/structs/structpath/path.go @@ -11,10 +11,9 @@ import ( ) const ( - tagStruct = -1 - tagMapKey = -2 - tagDotStar = -4 - tagBracketStar = -5 + tagStringKey = -1 + tagDotStar = -2 + tagBracketStar = -3 ) // PathNode represents a node in a path for struct diffing. @@ -23,7 +22,7 @@ type PathNode struct { prev *PathNode key string // Computed key (JSON key for structs, string key for maps, or Go field name for fallback) // If index >= 0, the node specifies a slice/array index in index. - // If index < 0, this describes the type of node (see tagStruct and other consts above) + // If index < 0, this describes the type of node index int } @@ -41,16 +40,6 @@ func (p *PathNode) Index() (int, bool) { return -1, false } -func (p *PathNode) MapKey() (string, bool) { - if p == nil { - return "", false - } - if p.index == tagMapKey { - return p.key, true - } - return "", false -} - func (p *PathNode) DotStar() bool { if p == nil { return false @@ -65,22 +54,12 @@ func (p *PathNode) BracketStar() bool { return p.index == tagBracketStar } -func (p *PathNode) Field() (string, bool) { - if p == nil { - return "", false - } - if p.index == tagStruct { - return p.key, true - } - return "", false -} - // StringKey returns either Field() or MapKey() if either is available func (p *PathNode) StringKey() (string, bool) { if p == nil { return "", false } - if p.index == tagStruct || p.index == tagMapKey { + if p.index == tagStringKey { return p.key, true } return "", false @@ -123,32 +102,13 @@ func NewIndex(prev *PathNode, index int) *PathNode { } } -// NewMapKey creates a new PathNode for a map key. -func NewMapKey(prev *PathNode, key string) *PathNode { - return &PathNode{ - prev: prev, - key: key, - index: tagMapKey, - } -} - -// NewStructField creates a new PathNode for a struct field. +// NewStringKey creates either StructField or MapKey // The fieldName should be the resolved field name (e.g., from JSON tag or Go field name). -func NewStructField(prev *PathNode, fieldName string) *PathNode { +func NewStringKey(prev *PathNode, fieldName string) *PathNode { return &PathNode{ prev: prev, key: fieldName, - index: tagStruct, - } -} - -// NewStringKey creates either StructField or MapKey -// The fieldName should be the resolved field name (e.g., from JSON tag or Go field name). -func NewStringKey(prev *PathNode, fieldName string) *PathNode { - if isValidField(fieldName) { - return NewStructField(prev, fieldName) - } else { - return NewMapKey(prev, fieldName) + index: tagStringKey, } } @@ -198,7 +158,7 @@ func (p *PathNode) String() string { return prev + "[*]" } - if p.index == tagStruct { + if isValidField(p.key) { prev := p.prev.String() if prev == "" { return p.key @@ -296,11 +256,11 @@ func Parse(s string) (*PathNode, error) { case stateField: if ch == '.' { - result = NewStructField(result, currentToken.String()) + result = NewStringKey(result, currentToken.String()) currentToken.Reset() state = stateFieldStart } else if ch == '[' { - result = NewStructField(result, currentToken.String()) + result = NewStringKey(result, currentToken.String()) currentToken.Reset() state = stateBracketOpen } else if !isReservedFieldChar(ch) { @@ -364,7 +324,7 @@ func Parse(s string) (*PathNode, error) { state = stateMapKey case ']': // End of map key - result = NewMapKey(result, currentToken.String()) + result = NewStringKey(result, currentToken.String()) currentToken.Reset() state = stateExpectDotOrEnd default: @@ -404,7 +364,7 @@ func Parse(s string) (*PathNode, error) { case stateStart: return result, nil // Empty path, result is nil case stateField: - result = NewStructField(result, currentToken.String()) + result = NewStringKey(result, currentToken.String()) return result, nil case stateDotStar: result = NewDotStar(result) diff --git a/libs/structs/structpath/path_test.go b/libs/structs/structpath/path_test.go index a222ccbae8..e7940228de 100644 --- a/libs/structs/structpath/path_test.go +++ b/libs/structs/structpath/path_test.go @@ -12,8 +12,7 @@ func TestPathNode(t *testing.T) { node *PathNode String string Index any - MapKey any - Field any + StringKey any Root any DotStar bool BracketStar bool @@ -32,34 +31,10 @@ func TestPathNode(t *testing.T) { Index: 5, }, { - name: "map key", - node: NewMapKey(nil, "mykey"), - String: `['mykey']`, - MapKey: "mykey", - }, - { - name: "struct field with JSON tag", - node: NewStructField(nil, "json_name"), - String: "json_name", - Field: "json_name", - }, - { - name: "struct field without JSON tag (fallback to Go name)", - node: NewStructField(nil, "GoFieldName"), - String: "GoFieldName", - Field: "GoFieldName", - }, - { - name: "struct field with dash JSON tag", - node: NewStructField(nil, "-"), - String: "-", - Field: "-", - }, - { - name: "struct field with JSON tag options", - node: NewStructField(nil, "lazy_field"), - String: "lazy_field", - Field: "lazy_field", + name: "map key", + node: NewStringKey(nil, "mykey"), + String: `mykey`, + StringKey: "mykey", }, { name: "dot star", @@ -77,151 +52,126 @@ func TestPathNode(t *testing.T) { // Two node tests { name: "struct field -> array index", - node: NewIndex(NewStructField(nil, "items"), 3), + node: NewIndex(NewStringKey(nil, "items"), 3), String: "items[3]", Index: 3, }, { - name: "struct field -> map key", - node: NewMapKey(NewStructField(nil, "config"), "database"), - String: `config['database']`, - MapKey: "database", + name: "struct field -> map key", + node: NewStringKey(NewStringKey(nil, "config"), "database.name"), + String: `config['database.name']`, + StringKey: "database.name", }, { - name: "struct field -> struct field", - node: NewStructField(NewStructField(nil, "user"), "name"), - String: "user.name", - Field: "name", + name: "struct field -> struct field", + node: NewStringKey(NewStringKey(nil, "user"), "name"), + String: "user.name", + StringKey: "name", }, { name: "map key -> array index", - node: NewIndex(NewMapKey(nil, "servers"), 0), - String: `['servers'][0]`, + node: NewIndex(NewStringKey(nil, "servers list"), 0), + String: `['servers list'][0]`, Index: 0, }, { - name: "map key -> struct field", - node: NewStructField(NewMapKey(nil, "primary"), "host"), - String: `['primary'].host`, - Field: "host", - }, - { - name: "array index -> struct field", - node: NewStructField(NewIndex(nil, 2), "id"), - String: "[2].id", - Field: "id", + name: "array index -> struct field", + node: NewStringKey(NewIndex(nil, 2), "id"), + String: "[2].id", + StringKey: "id", }, { - name: "array index -> map key", - node: NewMapKey(NewIndex(nil, 1), "status"), - String: `[1]['status']`, - MapKey: "status", - }, - { - name: "struct field without JSON tag -> struct field with JSON tag", - node: NewStructField(NewStructField(nil, "Parent"), "child_name"), - String: "Parent.child_name", - Field: "child_name", + name: "array index -> map key", + node: NewStringKey(NewIndex(nil, 1), "status{}"), + String: `[1]['status{}']`, + StringKey: "status{}", }, { name: "dot star with parent", - node: NewDotStar(NewStructField(nil, "Parent")), + node: NewDotStar(NewStringKey(nil, "Parent")), String: "Parent.*", DotStar: true, }, { name: "bracket star with parent", - node: NewBracketStar(NewStructField(nil, "Parent")), + node: NewBracketStar(NewStringKey(nil, "Parent")), String: "Parent[*]", BracketStar: true, }, // Edge cases with special characters in map keys { - name: "map key with single quote", - node: NewMapKey(nil, "key's"), - String: `['key''s']`, - MapKey: "key's", + name: "map key with single quote", + node: NewStringKey(nil, "key's"), + String: `['key''s']`, + StringKey: "key's", }, { - name: "map key with multiple single quotes", - node: NewMapKey(nil, "''"), - String: `['''''']`, - MapKey: "''", + name: "map key with multiple single quotes", + node: NewStringKey(nil, "''"), + String: `['''''']`, + StringKey: "''", }, { - name: "empty map key", - node: NewMapKey(nil, ""), - String: `['']`, - MapKey: "", + name: "empty map key", + node: NewStringKey(nil, ""), + String: `['']`, + StringKey: "", }, { name: "complex path", - node: NewStructField( + node: NewStringKey( NewIndex( - NewMapKey( - NewStructField( - NewStructField(nil, "user"), + NewStringKey( + NewStringKey( + NewStringKey(nil, "user"), "settings"), - "theme"), + "theme.list"), 0), "color"), - String: "user.settings['theme'][0].color", - Field: "color", + String: "user.settings['theme.list'][0].color", + StringKey: "color", }, { - name: "field with special characters", - node: NewStructField(nil, "field@name:with#symbols!"), - String: "field@name:with#symbols!", - Field: "field@name:with#symbols!", + name: "field with special characters", + node: NewStringKey(nil, "field@name:with#symbols!"), + String: "field@name:with#symbols!", + StringKey: "field@name:with#symbols!", }, { - name: "field with spaces", - node: NewStringKey(nil, "field with spaces"), - String: "['field with spaces']", - MapKey: "field with spaces", + name: "field with spaces", + node: NewStringKey(nil, "field with spaces"), + String: "['field with spaces']", + StringKey: "field with spaces", }, { - name: "field starting with digit", - node: NewStructField(nil, "123field"), - String: "123field", - Field: "123field", + name: "field starting with digit", + node: NewStringKey(nil, "123field"), + String: "123field", + StringKey: "123field", }, { - name: "field with unicode", - node: NewStructField(nil, "名前🙂"), - String: "名前🙂", - Field: "名前🙂", + name: "field with unicode", + node: NewStringKey(nil, "名前🙂"), + String: "名前🙂", + StringKey: "名前🙂", }, { - name: "map key with reserved characters", - node: NewMapKey(nil, "key\x00[],`"), - String: "['key\x00[],`']", - MapKey: "key\x00[],`", + name: "map key with reserved characters", + node: NewStringKey(nil, "key\x00[],`"), + String: "['key\x00[],`']", + StringKey: "key\x00[],`", }, - // Additional dot-star pattern tests - { - name: "field dot star", - node: NewDotStar(NewStructField(nil, "bla")), - String: "bla.*", - DotStar: true, - }, - { - name: "field dot star dot field", - node: NewStructField(NewDotStar(NewStructField(nil, "bla")), "foo"), - String: "bla.*.foo", - Field: "foo", - }, { name: "field dot star bracket index", - node: NewIndex(NewDotStar(NewStructField(nil, "bla")), 0), + node: NewIndex(NewDotStar(NewStringKey(nil, "bla")), 0), String: "bla.*[0]", Index: 0, }, { name: "field dot star bracket star", - node: NewBracketStar(NewDotStar(NewStructField(nil, "bla"))), + node: NewBracketStar(NewDotStar(NewStringKey(nil, "bla"))), String: "bla.*[*]", BracketStar: true, }, @@ -252,26 +202,14 @@ func TestPathNode(t *testing.T) { assert.True(t, isIndex) } - // Field - gotField, isField := tt.node.Field() - if tt.Field == nil { - assert.Equal(t, "", gotField) - assert.False(t, isField) - } else { - expected := tt.Field.(string) - assert.Equal(t, expected, gotField) - assert.True(t, isField) - } - - // MapKey - gotMapKey, isMapKey := tt.node.MapKey() - if tt.MapKey == nil { - assert.Equal(t, "", gotMapKey) - assert.False(t, isMapKey) + gotStringKey, isStringKey := tt.node.StringKey() + if tt.StringKey == nil { + assert.Equal(t, "", gotStringKey) + assert.False(t, isStringKey) } else { - expected := tt.MapKey.(string) - assert.Equal(t, expected, gotMapKey) - assert.True(t, isMapKey) + expected := tt.StringKey.(string) + assert.Equal(t, expected, gotStringKey) + assert.True(t, isStringKey) } // IsRoot diff --git a/libs/structs/structwalk/walk.go b/libs/structs/structwalk/walk.go index 5fce805ad3..0adb1cf6df 100644 --- a/libs/structs/structwalk/walk.go +++ b/libs/structs/structwalk/walk.go @@ -92,7 +92,7 @@ func walkValue(path *structpath.PathNode, val reflect.Value, field *reflect.Stru sort.Strings(keys) for _, ks := range keys { v := val.MapIndex(reflect.ValueOf(ks)) - node := structpath.NewMapKey(path, ks) + node := structpath.NewStringKey(path, ks) walkValue(node, v, nil, visit) } diff --git a/libs/structs/structwalk/walk_test.go b/libs/structs/structwalk/walk_test.go index b36960e5fe..f65f9ab5ae 100644 --- a/libs/structs/structwalk/walk_test.go +++ b/libs/structs/structwalk/walk_test.go @@ -87,8 +87,8 @@ func TestValueJobSettings(t *testing.T) { } assert.Equal(t, map[string]any{ - `tags['env']`: "test", - `tags['team']`: "data", + `tags.env`: "test", + `tags.team`: "data", "name": "test-job", "max_concurrent_runs": 5, "timeout_seconds": 3600, From 0e72ebc6dcdb772edbb89c72702d56182f9e8f30 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 22 Sep 2025 14:55:34 +0200 Subject: [PATCH 11/20] clean up --- libs/structs/structaccess/get.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/libs/structs/structaccess/get.go b/libs/structs/structaccess/get.go index 4817907567..18ba74082a 100644 --- a/libs/structs/structaccess/get.go +++ b/libs/structs/structaccess/get.go @@ -25,10 +25,7 @@ func GetByString(v any, path string) (any, error) { } // Get returns the value at the given path inside v. -// - For structs: supports both .field and ['field'] notation -// - For maps: supports both ['key'] and .key notation -// - For slices/arrays: an index [N] selects the N-th element. -// - Wildcards ("*" or "[*]") are not supported and return an error. +// Wildcards ("*" or "[*]") are not supported and return an error. func Get(v any, path *structpath.PathNode) (any, error) { if path.IsRoot() { return v, nil From 148522c0e5b12501182692a6b242d17cb604ba5e Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 22 Sep 2025 14:56:17 +0200 Subject: [PATCH 12/20] clean up --- libs/structs/structaccess/typecheck.go | 1 - 1 file changed, 1 deletion(-) diff --git a/libs/structs/structaccess/typecheck.go b/libs/structs/structaccess/typecheck.go index 292d98ec89..58ab950a2f 100644 --- a/libs/structs/structaccess/typecheck.go +++ b/libs/structs/structaccess/typecheck.go @@ -42,7 +42,6 @@ func Validate(t reflect.Type, path *structpath.PathNode) error { cur = cur.Elem() } - // Handle different node types if _, isIndex := node.Index(); isIndex { // Index access: slice/array kind := cur.Kind() From 64117e1225776338ee39b134c979877f667c38e8 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 22 Sep 2025 14:58:27 +0200 Subject: [PATCH 13/20] clean up --- libs/structs/structaccess/typecheck.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/libs/structs/structaccess/typecheck.go b/libs/structs/structaccess/typecheck.go index 58ab950a2f..3f851571be 100644 --- a/libs/structs/structaccess/typecheck.go +++ b/libs/structs/structaccess/typecheck.go @@ -111,16 +111,6 @@ func FindStructFieldByKeyType(t reflect.Type, key string) (reflect.StructField, } return sf, t, true } - - // Fallback to Go field name when no JSON tag - if name == "" && sf.Name == key { - // Skip fields marked as internal/readonly - btag := structtag.BundleTag(sf.Tag.Get("bundle")) - if btag.Internal() || btag.ReadOnly() { - continue - } - return sf, t, true - } } // Second pass: search embedded anonymous structs recursively (flattening semantics) From daee8d6b371af616cc942999fd54c856957ac238 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 22 Sep 2025 15:02:16 +0200 Subject: [PATCH 14/20] iterative String() --- libs/structs/structpath/path.go | 64 +++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/libs/structs/structpath/path.go b/libs/structs/structpath/path.go index 22ec85fa39..0016625c34 100644 --- a/libs/structs/structpath/path.go +++ b/libs/structs/structpath/path.go @@ -127,7 +127,7 @@ func NewBracketStar(prev *PathNode) *PathNode { } // String returns the string representation of the path. -// The map keys are encoded in single quotes: tags['name']. Single quote can escaped by placing two single quotes: tags[””] (map key is one single quote). +// The map keys are encoded in single quotes: tags['name']. Single quote can escaped by placing two single quotes. // This encoding is chosen over traditional double quotes because when encoded in JSON it does not need to be escaped: // // { @@ -138,36 +138,48 @@ func (p *PathNode) String() string { return "" } - if p.index >= 0 { - return p.prev.String() + "[" + strconv.Itoa(p.index) + "]" - } + // Get all path components from root to current + components := p.AsSlice() - if p.index == tagDotStar { - prev := p.prev.String() - if prev == "" { - return "*" - } - return prev + ".*" - } - - if p.index == tagBracketStar { - prev := p.prev.String() - if prev == "" { - return "[*]" - } - return prev + "[*]" - } + var result strings.Builder - if isValidField(p.key) { - prev := p.prev.String() - if prev == "" { - return p.key + for i, node := range components { + if node.index >= 0 { + // Array/slice index + result.WriteString("[") + result.WriteString(strconv.Itoa(node.index)) + result.WriteString("]") + } else if node.index == tagDotStar { + // Dot star wildcard + if i == 0 { + result.WriteString("*") + } else { + result.WriteString(".*") + } + } else if node.index == tagBracketStar { + // Bracket star wildcard + if i == 0 { + result.WriteString("[*]") + } else { + result.WriteString("[*]") + } + } else if isValidField(node.key) { + // Valid field name + if i == 0 { + result.WriteString(node.key) + } else { + result.WriteString(".") + result.WriteString(node.key) + } + } else { + // Map key with single quotes + result.WriteString("[") + result.WriteString(EncodeMapKey(node.key)) + result.WriteString("]") } - return prev + "." + p.key } - // Format map key with single quotes, escaping single quotes by doubling them - return fmt.Sprintf("%s[%s]", p.prev.String(), EncodeMapKey(p.key)) + return result.String() } func EncodeMapKey(s string) string { From 407033a3503eed4c2f59f76c62f6c8828458dacb Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 22 Sep 2025 15:17:33 +0200 Subject: [PATCH 15/20] clean up --- libs/structs/structpath/path.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/libs/structs/structpath/path.go b/libs/structs/structpath/path.go index 0016625c34..23ae35803d 100644 --- a/libs/structs/structpath/path.go +++ b/libs/structs/structpath/path.go @@ -75,13 +75,10 @@ func (p *PathNode) Parent() *PathNode { // AsSlice returns the path as a slice of PathNodes from root to current. // Efficiently pre-allocates the exact length and fills in reverse order. func (p *PathNode) AsSlice() []*PathNode { - // Use Len() to get the length efficiently length := p.Len() - - // Allocate slice with exact capacity segments := make([]*PathNode, length) - // Fill in reverse order (from end to start) + // Fill in reverse order current := p for i := length - 1; i >= 0; i-- { segments[i] = current From c7daaab0bc10802cf703e9dc395fd3fa84acbdbf Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 22 Sep 2025 15:19:06 +0200 Subject: [PATCH 16/20] clean up --- libs/structs/structpath/path.go | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/libs/structs/structpath/path.go b/libs/structs/structpath/path.go index 23ae35803d..8fcb7fea32 100644 --- a/libs/structs/structpath/path.go +++ b/libs/structs/structpath/path.go @@ -147,27 +147,19 @@ func (p *PathNode) String() string { result.WriteString(strconv.Itoa(node.index)) result.WriteString("]") } else if node.index == tagDotStar { - // Dot star wildcard if i == 0 { result.WriteString("*") } else { result.WriteString(".*") } } else if node.index == tagBracketStar { - // Bracket star wildcard - if i == 0 { - result.WriteString("[*]") - } else { - result.WriteString("[*]") - } + result.WriteString("[*]") } else if isValidField(node.key) { // Valid field name - if i == 0 { - result.WriteString(node.key) - } else { + if i != 0 { result.WriteString(".") - result.WriteString(node.key) } + result.WriteString(node.key) } else { // Map key with single quotes result.WriteString("[") From 7fbf65fb60957b1cfcce289881bc3ff11fb3350c Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 22 Sep 2025 15:24:29 +0200 Subject: [PATCH 17/20] clean up --- libs/structs/structaccess/get_test.go | 63 +++++---------------------- 1 file changed, 10 insertions(+), 53 deletions(-) diff --git a/libs/structs/structaccess/get_test.go b/libs/structs/structaccess/get_test.go index 2d525027c5..2460b32a4f 100644 --- a/libs/structs/structaccess/get_test.go +++ b/libs/structs/structaccess/get_test.go @@ -137,6 +137,16 @@ func runCommonTests(t *testing.T, obj any) { path: "alias_map.foo", want: "bar", }, + { + name: "struct field with bracket notation", + path: "['connection']['id']", + want: "abc", + }, + { + name: "map key with bracket notation", + path: "labels['env']", + want: "dev", + }, // Regular scalar fields - always return zero values { @@ -398,56 +408,3 @@ func TestGet_BundleTag_SkipsPromoted(t *testing.T) { require.EqualError(t, ValidateByString(reflect.TypeOf(host{}), "hidden"), "hidden: field \"hidden\" not found in structaccess.host") } -func TestGet_FlexibleNotation(t *testing.T) { - type testStruct struct { - Field string `json:"field"` - Map map[string]string `json:"map"` - } - - obj := testStruct{ - Field: "value", - Map: map[string]string{"key": "mapvalue"}, - } - - tests := []struct { - name string - path string - want string - errFmt string - }{ - // Struct field access - both notations should work - { - name: "struct field with dot notation", - path: "field", - want: "value", - }, - { - name: "struct field with bracket notation", - path: "['field']", - want: "value", - }, - // Map key access - both notations should work - { - name: "map key with bracket notation", - path: "map['key']", - want: "mapvalue", - }, - { - name: "map key with dot notation", - path: "map.key", - want: "mapvalue", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := GetByString(obj, tt.path) - if tt.errFmt != "" { - require.EqualError(t, err, tt.errFmt) - return - } - require.NoError(t, err) - require.Equal(t, tt.want, got) - }) - } -} From ea558348fc14a60755950c3bcc8f9bc5912bc5ec Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 22 Sep 2025 15:30:12 +0200 Subject: [PATCH 18/20] lint fix --- libs/structs/structaccess/get_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/libs/structs/structaccess/get_test.go b/libs/structs/structaccess/get_test.go index 2460b32a4f..dec25b87b8 100644 --- a/libs/structs/structaccess/get_test.go +++ b/libs/structs/structaccess/get_test.go @@ -407,4 +407,3 @@ func TestGet_BundleTag_SkipsPromoted(t *testing.T) { require.EqualError(t, err, "hidden: field \"hidden\" not found in structaccess.host") require.EqualError(t, ValidateByString(reflect.TypeOf(host{}), "hidden"), "hidden: field \"hidden\" not found in structaccess.host") } - From 1b1939465d917b931bbe86cedeaa10655fb675d6 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 23 Sep 2025 21:13:00 +0200 Subject: [PATCH 19/20] feedback: add comments, ReverseInPlace --- libs/structs/structpath/path.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/libs/structs/structpath/path.go b/libs/structs/structpath/path.go index 8fcb7fea32..a656565b2b 100644 --- a/libs/structs/structpath/path.go +++ b/libs/structs/structpath/path.go @@ -11,8 +11,13 @@ import ( ) const ( - tagStringKey = -1 - tagDotStar = -2 + // Encodes string key, which is encoded as .field or as ['spark.conf'] + tagStringKey = -1 + + // Encodes wildcard after a dot: foo.* + tagDotStar = -2 + + // Encodes wildcard in brackets: foo[*] tagBracketStar = -3 ) @@ -124,7 +129,8 @@ func NewBracketStar(prev *PathNode) *PathNode { } // String returns the string representation of the path. -// The map keys are encoded in single quotes: tags['name']. Single quote can escaped by placing two single quotes. +// The string keys are encoded in dot syntax (foo.bar) if they don't have any reserved characters (so can be parsed as fields). +// Otherwise they are encoded in brackets + single quotes: tags['name']. Single quote can escaped by placing two single quotes. // This encoding is chosen over traditional double quotes because when encoded in JSON it does not need to be escaped: // // { @@ -473,11 +479,11 @@ func (p *PathNode) SkipPrefix(n int) *PathNode { current = current.Parent() } - return result.Reverse() + return result.ReverseInPlace() } -// Reverse returns a new PathNode with the order of components reversed. -func (p *PathNode) Reverse() *PathNode { +// ReverseInPlace returns a new PathNode with the order of components reversed. +func (p *PathNode) ReverseInPlace() *PathNode { var result *PathNode current := p for current != nil { From e627b0e02341e136be0602f000ae0bdb29d69b36 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 23 Sep 2025 21:25:05 +0200 Subject: [PATCH 20/20] add test for PureReferenceToPath --- libs/structs/structpath/path_test.go | 49 ++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/libs/structs/structpath/path_test.go b/libs/structs/structpath/path_test.go index e7940228de..a9e19501dc 100644 --- a/libs/structs/structpath/path_test.go +++ b/libs/structs/structpath/path_test.go @@ -508,3 +508,52 @@ func TestLen(t *testing.T) { }) } } + +func TestPureReferenceToPath(t *testing.T) { + tests := []struct { + name string + input string + expected string + ok bool + }{ + { + name: "simple reference", + input: "${resources.jobs.foo.id}", + expected: "resources.jobs.foo.id", + ok: true, + }, + { + name: "simple reference", + input: "${resources.jobs.foo.tasks[1].env.key}", + expected: "resources.jobs.foo.tasks[1].env.key", + ok: true, + }, + { + name: "complex nested reference", + input: "${var.resources.jobs['my_job'].tasks[0]}", + // we use regex from dyn module which only support integers inside brackets: + // expected: "resources.jobs['my_job'].tasks[0]", + }, + { + name: "not a pure reference", + input: "prefix_${var.field}", + }, + { + name: "not a variable reference", + input: "plain_string", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pathNode, ok := PureReferenceToPath(tt.input) + assert.Equal(t, tt.ok, ok) + if tt.ok { + assert.NotNil(t, pathNode) + assert.Equal(t, tt.expected, pathNode.String()) + } else { + assert.Nil(t, pathNode) + } + }) + } +}