From 2fd8f7e26699d1e0b51ff79e4630779ef97399ea Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Thu, 15 Feb 2024 15:42:24 +0100 Subject: [PATCH 1/2] Add option to include fields present in the type but not in the value This feature supports variable lookups in a `dyn.Value` that are present in the type but haven't been initialized with a value. For example: `${bundle.git.origin_url}` is present in the `dyn.Value` only if it was assigned a value. If it wasn't assigned a value it should resolve to the empty string. This normalization option, when set, ensures that all fields that are represented in the specified type are present in the return value. --- libs/dyn/convert/normalize.go | 100 +++++++++++++++++++++++------ libs/dyn/convert/normalize_test.go | 47 ++++++++++++++ 2 files changed, 127 insertions(+), 20 deletions(-) diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go index 5595aae1e9..989ef8dd8a 100644 --- a/libs/dyn/convert/normalize.go +++ b/libs/dyn/convert/normalize.go @@ -9,30 +9,51 @@ import ( "github.com/databricks/cli/libs/dyn" ) -func Normalize(dst any, src dyn.Value) (dyn.Value, diag.Diagnostics) { - return normalizeType(reflect.TypeOf(dst), src) +// NormalizeOption is the type for options that can be passed to Normalize. +type NormalizeOption int + +const ( + // IncludeMissingFields causes the normalization to include fields that defined on the given + // type but are missing in the source value. They are included with their zero values. + IncludeMissingFields NormalizeOption = iota +) + +type normalizeOptions struct { + includeMissingFields bool +} + +func Normalize(dst any, src dyn.Value, opts ...NormalizeOption) (dyn.Value, diag.Diagnostics) { + var n normalizeOptions + for _, opt := range opts { + switch opt { + case IncludeMissingFields: + n.includeMissingFields = true + } + } + + return n.normalizeType(reflect.TypeOf(dst), src) } -func normalizeType(typ reflect.Type, src dyn.Value) (dyn.Value, diag.Diagnostics) { +func (n normalizeOptions) normalizeType(typ reflect.Type, src dyn.Value) (dyn.Value, diag.Diagnostics) { for typ.Kind() == reflect.Pointer { typ = typ.Elem() } switch typ.Kind() { case reflect.Struct: - return normalizeStruct(typ, src) + return n.normalizeStruct(typ, src) case reflect.Map: - return normalizeMap(typ, src) + return n.normalizeMap(typ, src) case reflect.Slice: - return normalizeSlice(typ, src) + return n.normalizeSlice(typ, src) case reflect.String: - return normalizeString(typ, src) + return n.normalizeString(typ, src) case reflect.Bool: - return normalizeBool(typ, src) + return n.normalizeBool(typ, src) case reflect.Int, reflect.Int32, reflect.Int64: - return normalizeInt(typ, src) + return n.normalizeInt(typ, src) case reflect.Float32, reflect.Float64: - return normalizeFloat(typ, src) + return n.normalizeFloat(typ, src) } return dyn.InvalidValue, diag.Errorf("unsupported type: %s", typ.Kind()) @@ -46,7 +67,7 @@ func typeMismatch(expected dyn.Kind, src dyn.Value) diag.Diagnostic { } } -func normalizeStruct(typ reflect.Type, src dyn.Value) (dyn.Value, diag.Diagnostics) { +func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value) (dyn.Value, diag.Diagnostics) { var diags diag.Diagnostics switch src.Kind() { @@ -65,7 +86,7 @@ func normalizeStruct(typ reflect.Type, src dyn.Value) (dyn.Value, diag.Diagnosti } // Normalize the value according to the field type. - v, err := normalizeType(typ.FieldByIndex(index).Type, v) + v, err := n.normalizeType(typ.FieldByIndex(index).Type, v) if err != nil { diags = diags.Extend(err) // Skip the element if it cannot be normalized. @@ -77,6 +98,45 @@ func normalizeStruct(typ reflect.Type, src dyn.Value) (dyn.Value, diag.Diagnosti out[k] = v } + // Return the normalized value if missing fields are not included. + if !n.includeMissingFields { + return dyn.NewValue(out, src.Location()), diags + } + + // Populate missing fields with their zero values. + for k, index := range info.Fields { + if _, ok := out[k]; ok { + continue + } + + // Optionally dereference pointers to get the underlying field type. + ftyp := typ.FieldByIndex(index).Type + for ftyp.Kind() == reflect.Pointer { + ftyp = ftyp.Elem() + } + + var v dyn.Value + switch ftyp.Kind() { + case reflect.Struct, reflect.Map: + v, _ = n.normalizeType(ftyp, dyn.V(map[string]dyn.Value{})) + case reflect.Slice: + v, _ = n.normalizeType(ftyp, dyn.V([]dyn.Value{})) + case reflect.String: + v, _ = n.normalizeType(ftyp, dyn.V("")) + case reflect.Bool: + v, _ = n.normalizeType(ftyp, dyn.V(false)) + case reflect.Int, reflect.Int32, reflect.Int64: + v, _ = n.normalizeType(ftyp, dyn.V(int64(0))) + case reflect.Float32, reflect.Float64: + v, _ = n.normalizeType(ftyp, dyn.V(float64(0))) + default: + panic(fmt.Sprintf("unsupported type: %s", ftyp.Kind())) + } + if v.IsValid() { + out[k] = v + } + } + return dyn.NewValue(out, src.Location()), diags case dyn.KindNil: return src, diags @@ -85,7 +145,7 @@ func normalizeStruct(typ reflect.Type, src dyn.Value) (dyn.Value, diag.Diagnosti return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindMap, src)) } -func normalizeMap(typ reflect.Type, src dyn.Value) (dyn.Value, diag.Diagnostics) { +func (n normalizeOptions) normalizeMap(typ reflect.Type, src dyn.Value) (dyn.Value, diag.Diagnostics) { var diags diag.Diagnostics switch src.Kind() { @@ -93,7 +153,7 @@ func normalizeMap(typ reflect.Type, src dyn.Value) (dyn.Value, diag.Diagnostics) out := make(map[string]dyn.Value) for k, v := range src.MustMap() { // Normalize the value according to the map element type. - v, err := normalizeType(typ.Elem(), v) + v, err := n.normalizeType(typ.Elem(), v) if err != nil { diags = diags.Extend(err) // Skip the element if it cannot be normalized. @@ -113,7 +173,7 @@ func normalizeMap(typ reflect.Type, src dyn.Value) (dyn.Value, diag.Diagnostics) return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindMap, src)) } -func normalizeSlice(typ reflect.Type, src dyn.Value) (dyn.Value, diag.Diagnostics) { +func (n normalizeOptions) normalizeSlice(typ reflect.Type, src dyn.Value) (dyn.Value, diag.Diagnostics) { var diags diag.Diagnostics switch src.Kind() { @@ -121,7 +181,7 @@ func normalizeSlice(typ reflect.Type, src dyn.Value) (dyn.Value, diag.Diagnostic out := make([]dyn.Value, 0, len(src.MustSequence())) for _, v := range src.MustSequence() { // Normalize the value according to the slice element type. - v, err := normalizeType(typ.Elem(), v) + v, err := n.normalizeType(typ.Elem(), v) if err != nil { diags = diags.Extend(err) // Skip the element if it cannot be normalized. @@ -141,7 +201,7 @@ func normalizeSlice(typ reflect.Type, src dyn.Value) (dyn.Value, diag.Diagnostic return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindSequence, src)) } -func normalizeString(typ reflect.Type, src dyn.Value) (dyn.Value, diag.Diagnostics) { +func (n normalizeOptions) normalizeString(typ reflect.Type, src dyn.Value) (dyn.Value, diag.Diagnostics) { var diags diag.Diagnostics var out string @@ -161,7 +221,7 @@ func normalizeString(typ reflect.Type, src dyn.Value) (dyn.Value, diag.Diagnosti return dyn.NewValue(out, src.Location()), diags } -func normalizeBool(typ reflect.Type, src dyn.Value) (dyn.Value, diag.Diagnostics) { +func (n normalizeOptions) normalizeBool(typ reflect.Type, src dyn.Value) (dyn.Value, diag.Diagnostics) { var diags diag.Diagnostics var out bool @@ -186,7 +246,7 @@ func normalizeBool(typ reflect.Type, src dyn.Value) (dyn.Value, diag.Diagnostics return dyn.NewValue(out, src.Location()), diags } -func normalizeInt(typ reflect.Type, src dyn.Value) (dyn.Value, diag.Diagnostics) { +func (n normalizeOptions) normalizeInt(typ reflect.Type, src dyn.Value) (dyn.Value, diag.Diagnostics) { var diags diag.Diagnostics var out int64 @@ -210,7 +270,7 @@ func normalizeInt(typ reflect.Type, src dyn.Value) (dyn.Value, diag.Diagnostics) return dyn.NewValue(out, src.Location()), diags } -func normalizeFloat(typ reflect.Type, src dyn.Value) (dyn.Value, diag.Diagnostics) { +func (n normalizeOptions) normalizeFloat(typ reflect.Type, src dyn.Value) (dyn.Value, diag.Diagnostics) { var diags diag.Diagnostics var out float64 diff --git a/libs/dyn/convert/normalize_test.go b/libs/dyn/convert/normalize_test.go index 7028161556..d59cc3b351 100644 --- a/libs/dyn/convert/normalize_test.go +++ b/libs/dyn/convert/normalize_test.go @@ -142,6 +142,53 @@ func TestNormalizeStructNestedError(t *testing.T) { ) } +func TestNormalizeStructIncludeMissingFields(t *testing.T) { + type Nested struct { + String string `json:"string"` + } + + type Tmp struct { + // Verify that fields that are already set in the dynamic value are not overridden. + Existing string `json:"existing"` + + // Verify that structs are recursively normalized if not set. + Nested Nested `json:"nested"` + Ptr *Nested `json:"ptr"` + + // Verify that containers are also zero-initialized if not set. + Map map[string]string `json:"map"` + Slice []string `json:"slice"` + + // Verify that primitive types are zero-initialized if not set. + String string `json:"string"` + Bool bool `json:"bool"` + Int int `json:"int"` + Float float64 `json:"float"` + } + + var typ Tmp + vin := dyn.V(map[string]dyn.Value{ + "existing": dyn.V("already set"), + }) + vout, err := Normalize(typ, vin, IncludeMissingFields) + assert.Empty(t, err) + assert.Equal(t, dyn.V(map[string]dyn.Value{ + "existing": dyn.V("already set"), + "nested": dyn.V(map[string]dyn.Value{ + "string": dyn.V(""), + }), + "ptr": dyn.V(map[string]dyn.Value{ + "string": dyn.V(""), + }), + "map": dyn.V(map[string]dyn.Value{}), + "slice": dyn.V([]dyn.Value{}), + "string": dyn.V(""), + "bool": dyn.V(false), + "int": dyn.V(int64(0)), + "float": dyn.V(float64(0)), + }), vout) +} + func TestNormalizeMap(t *testing.T) { var typ map[string]string vin := dyn.V(map[string]dyn.Value{ From 8774a9684f44086cf60816cee6baed1db9aad319 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Thu, 15 Feb 2024 16:09:10 +0100 Subject: [PATCH 2/2] Skip instead of panic --- libs/dyn/convert/normalize.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go index 989ef8dd8a..26df09578d 100644 --- a/libs/dyn/convert/normalize.go +++ b/libs/dyn/convert/normalize.go @@ -125,12 +125,14 @@ func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value) (dyn. v, _ = n.normalizeType(ftyp, dyn.V("")) case reflect.Bool: v, _ = n.normalizeType(ftyp, dyn.V(false)) - case reflect.Int, reflect.Int32, reflect.Int64: + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: v, _ = n.normalizeType(ftyp, dyn.V(int64(0))) case reflect.Float32, reflect.Float64: v, _ = n.normalizeType(ftyp, dyn.V(float64(0))) default: - panic(fmt.Sprintf("unsupported type: %s", ftyp.Kind())) + // Skip fields for which we do not have a natural [dyn.Value] equivalent. + // For example, we don't handle reflect.Complex* and reflect.Uint* types. + continue } if v.IsValid() { out[k] = v