From bfe7034754737bf2dac386e1b41e575f615e4bcb Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 10 Apr 2024 10:48:33 +0200 Subject: [PATCH 1/2] Do not emit warning on YAML anchor blocks --- bundle/tests/loader.go | 15 +++++++++++--- .../databricks.yml | 15 ++++++++++++++ bundle/tests/yaml_anchors_test.go | 10 ++++++++++ libs/dyn/convert/normalize.go | 6 ++++++ libs/dyn/convert/normalize_test.go | 20 +++++++++++++++++++ 5 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 bundle/tests/yaml_anchors_separate_block/databricks.yml diff --git a/bundle/tests/loader.go b/bundle/tests/loader.go index e7cf18f732..8eddcf9a11 100644 --- a/bundle/tests/loader.go +++ b/bundle/tests/loader.go @@ -7,6 +7,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/phases" + "github.com/databricks/cli/libs/diag" "github.com/stretchr/testify/require" ) @@ -20,9 +21,18 @@ func load(t *testing.T, path string) *bundle.Bundle { } func loadTarget(t *testing.T, path, env string) *bundle.Bundle { + b, diags := loadTargetWithDiags(path, env) + require.NoError(t, diags.Error()) + return b +} + +func loadTargetWithDiags(path, env string) (*bundle.Bundle, diag.Diagnostics) { ctx := context.Background() b, err := bundle.Load(ctx, path) - require.NoError(t, err) + if err != nil { + return nil, diag.FromErr(err) + } + diags := bundle.Apply(ctx, b, bundle.Seq( phases.LoadNamedTarget(env), mutator.RewriteSyncPaths(), @@ -30,6 +40,5 @@ func loadTarget(t *testing.T, path, env string) *bundle.Bundle { mutator.MergeJobTasks(), mutator.MergePipelineClusters(), )) - require.NoError(t, diags.Error()) - return b + return b, diags } diff --git a/bundle/tests/yaml_anchors_separate_block/databricks.yml b/bundle/tests/yaml_anchors_separate_block/databricks.yml new file mode 100644 index 0000000000..447d5d0bb3 --- /dev/null +++ b/bundle/tests/yaml_anchors_separate_block/databricks.yml @@ -0,0 +1,15 @@ +bundle: + name: yaml_anchors_separate_block + +tags: &custom_tags + Tag1: "Value1" + Tag2: "Value2" + Tag3: "Value3" + +resources: + jobs: + my_job: + tasks: + - task_key: yaml_anchors_separate_block + tags: + <<: *custom_tags diff --git a/bundle/tests/yaml_anchors_test.go b/bundle/tests/yaml_anchors_test.go index 95cec30ad4..c9533e1ae8 100644 --- a/bundle/tests/yaml_anchors_test.go +++ b/bundle/tests/yaml_anchors_test.go @@ -22,3 +22,13 @@ func TestYAMLAnchors(t *testing.T) { assert.Equal(t, "10.4.x-scala2.12", t0.NewCluster.SparkVersion) assert.Equal(t, "10.4.x-scala2.12", t1.NewCluster.SparkVersion) } + +func TestYAMLAnchorsNoWarnings(t *testing.T) { + _, diags := loadTargetWithDiags("./yaml_anchors", "default") + assert.Empty(t, diags) +} + +func TestYAMLAnchorsSeparateBlockNoWarnings(t *testing.T) { + _, diags := loadTargetWithDiags("./yaml_anchors_separate_block", "default") + assert.Empty(t, diags) +} diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go index 296e2abb26..57360053ca 100644 --- a/libs/dyn/convert/normalize.go +++ b/libs/dyn/convert/normalize.go @@ -89,6 +89,12 @@ func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value, seen for _, pair := range src.MustMap().Pairs() { pk := pair.Key pv := pair.Value + + // Skip anchor fields. + if pv.IsAnchor() { + continue + } + index, ok := info.Fields[pk.MustString()] if !ok { diags = diags.Append(diag.Diagnostic{ diff --git a/libs/dyn/convert/normalize_test.go b/libs/dyn/convert/normalize_test.go index 133eaef8fc..1a0869a9fa 100644 --- a/libs/dyn/convert/normalize_test.go +++ b/libs/dyn/convert/normalize_test.go @@ -659,3 +659,23 @@ func TestNormalizeFloatError(t *testing.T) { Path: dyn.EmptyPath, }, err[0]) } + +func TestNormalizeAnchors(t *testing.T) { + type Tmp struct { + Foo string `json:"foo"` + } + + var typ Tmp + vin := dyn.V(map[string]dyn.Value{ + "foo": dyn.V("bar"), + "anchor": dyn.V("anchor").MarkAnchor(), + }) + + vout, err := Normalize(typ, vin) + assert.Len(t, err, 0) + + // The field that can be mapped to the struct field is retained. + assert.Equal(t, map[string]any{ + "foo": "bar", + }, vout.AsAny()) +} From c43859ab77a7871a4b04bdacd4c065b489473e31 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 10 Apr 2024 11:45:01 +0200 Subject: [PATCH 2/2] fix tests --- bundle/tests/yaml_anchors_test.go | 2 ++ libs/dyn/convert/normalize.go | 19 ++++++++----------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/bundle/tests/yaml_anchors_test.go b/bundle/tests/yaml_anchors_test.go index c9533e1ae8..5c84970514 100644 --- a/bundle/tests/yaml_anchors_test.go +++ b/bundle/tests/yaml_anchors_test.go @@ -19,6 +19,8 @@ func TestYAMLAnchors(t *testing.T) { require.NotNil(t, t0) require.NotNil(t, t1) + require.NotNil(t, t0.NewCluster) + require.NotNil(t, t1.NewCluster) assert.Equal(t, "10.4.x-scala2.12", t0.NewCluster.SparkVersion) assert.Equal(t, "10.4.x-scala2.12", t1.NewCluster.SparkVersion) } diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go index 57360053ca..b4bee9773f 100644 --- a/libs/dyn/convert/normalize.go +++ b/libs/dyn/convert/normalize.go @@ -90,19 +90,16 @@ func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value, seen pk := pair.Key pv := pair.Value - // Skip anchor fields. - if pv.IsAnchor() { - continue - } - index, ok := info.Fields[pk.MustString()] if !ok { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("unknown field: %s", pk.MustString()), - Location: pk.Location(), - Path: path, - }) + if !pv.IsAnchor() { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: fmt.Sprintf("unknown field: %s", pk.MustString()), + Location: pk.Location(), + Path: path, + }) + } continue }