From 07e8e2fe632b42f2134a5ed20097eedba639e8ac Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 13 Jun 2025 12:32:06 +0200 Subject: [PATCH 1/5] Fix resolution of lookup variables Fix warning to say that unresolved references may remain (but they also can be fully resolved, we only check that final round is reached). --- .../variables/complex-cycle-self/output.txt | 2 +- .../bundle/variables/complex-cycle/output.txt | 2 +- .../bundle/variables/global_vars.yml | 1 + .../variables/issue_3039/databricks.yml | 28 +++++++++++++++++++ acceptance/bundle/variables/issue_3039/script | 1 + .../bundle/variables/issue_3039/test.toml | 23 +++++++++++++++ .../mutator/resolve_variable_references.go | 17 +++++++---- 7 files changed, 66 insertions(+), 8 deletions(-) create mode 100644 acceptance/bundle/variables/issue_3039/bundle/variables/global_vars.yml create mode 100644 acceptance/bundle/variables/issue_3039/databricks.yml create mode 100644 acceptance/bundle/variables/issue_3039/script create mode 100644 acceptance/bundle/variables/issue_3039/test.toml diff --git a/acceptance/bundle/variables/complex-cycle-self/output.txt b/acceptance/bundle/variables/complex-cycle-self/output.txt index 7447de349c..83ce6da568 100644 --- a/acceptance/bundle/variables/complex-cycle-self/output.txt +++ b/acceptance/bundle/variables/complex-cycle-self/output.txt @@ -1,4 +1,4 @@ -Warning: Detected unresolved variables after 11 resolution rounds +Warning: Variables references are too deep, stopping resolution after 11 rounds. Unresolved variables may remain. Name: cycle Target: default diff --git a/acceptance/bundle/variables/complex-cycle/output.txt b/acceptance/bundle/variables/complex-cycle/output.txt index 7447de349c..83ce6da568 100644 --- a/acceptance/bundle/variables/complex-cycle/output.txt +++ b/acceptance/bundle/variables/complex-cycle/output.txt @@ -1,4 +1,4 @@ -Warning: Detected unresolved variables after 11 resolution rounds +Warning: Variables references are too deep, stopping resolution after 11 rounds. Unresolved variables may remain. Name: cycle Target: default diff --git a/acceptance/bundle/variables/issue_3039/bundle/variables/global_vars.yml b/acceptance/bundle/variables/issue_3039/bundle/variables/global_vars.yml new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/acceptance/bundle/variables/issue_3039/bundle/variables/global_vars.yml @@ -0,0 +1 @@ + diff --git a/acceptance/bundle/variables/issue_3039/databricks.yml b/acceptance/bundle/variables/issue_3039/databricks.yml new file mode 100644 index 0000000000..38f50f1832 --- /dev/null +++ b/acceptance/bundle/variables/issue_3039/databricks.yml @@ -0,0 +1,28 @@ +bundle: + name: issue-3039 + +variables: + tidal_service_account: + description: Gets tidal service account name/id. + lookup: + service_principal: "TIDALDBServAccount - ${var.uc_catalog}" + uc_catalog: + description: Unity Catalog prefix. + type: string + default: "" + +non_production_job_permissions: &non_prod_job_permissions + permissions: + - level: CAN_VIEW + service_principal_name: ${var.tidal_service_account} + +targets: + personal: + resources: + jobs: + xxx_job: + <<: *non_prod_job_permissions + variables: + uc_catalog: + description: Unity Catalog prefix. + default: "usdev" diff --git a/acceptance/bundle/variables/issue_3039/script b/acceptance/bundle/variables/issue_3039/script new file mode 100644 index 0000000000..72555b332a --- /dev/null +++ b/acceptance/bundle/variables/issue_3039/script @@ -0,0 +1 @@ +$CLI bundle validate diff --git a/acceptance/bundle/variables/issue_3039/test.toml b/acceptance/bundle/variables/issue_3039/test.toml new file mode 100644 index 0000000000..030d24eb28 --- /dev/null +++ b/acceptance/bundle/variables/issue_3039/test.toml @@ -0,0 +1,23 @@ +[[Server]] +Pattern = "GET /api/2.0/preview/scim/v2/ServicePrincipals" +Response.Body = '''{}''' # this body causes error, but works to check if resolution is done correctly + +# For some reason this body causes SDK to loop forever +#Response.Body = '''{ +# "Resources": [ +# { +# "displayName": "TIDALDBServAccount - usdev", +# "groups": [], +# "id": "10000", +# "applicationId": "e700887d-8550-4667-8884-dbc94808cfb6", +# "schemas": [ +# "urn:ietf:params:scim:schemas:core:2.0:ServicePrincipal" +# ], +# "active": true +# } +#], +#"totalResults": 1, +#"startIndex": 1, +#"itemsPerPage": 1, +#"schemas": [ +# "urn:ietf:params:scim:api:messages:2.0:ListResponse"]}''' diff --git a/bundle/config/mutator/resolve_variable_references.go b/bundle/config/mutator/resolve_variable_references.go index ea29cc71d4..a3c99e5bf3 100644 --- a/bundle/config/mutator/resolve_variable_references.go +++ b/bundle/config/mutator/resolve_variable_references.go @@ -66,11 +66,16 @@ func ResolveVariableReferencesWithoutResources(prefixes ...string) bundle.Mutato } func ResolveVariableReferencesInLookup() bundle.Mutator { - return &resolveVariableReferences{prefixes: []string{ - "bundle", - "workspace", - "variables", - }, pattern: dyn.NewPattern(dyn.Key("variables"), dyn.AnyKey(), dyn.Key("lookup")), lookupFn: lookupForVariables} + return &resolveVariableReferences{ + prefixes: []string{ + "bundle", + "workspace", + "variables", + }, + pattern: dyn.NewPattern(dyn.Key("variables"), dyn.AnyKey(), dyn.Key("lookup")), + lookupFn: lookupForVariables, + extraRounds: maxResolutionRounds - 1, + } } func lookup(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) { @@ -149,7 +154,7 @@ func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle) if round >= maxRounds-1 { diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, - Summary: fmt.Sprintf("Detected unresolved variables after %d resolution rounds", round+1), + Summary: fmt.Sprintf("Variables references are too deep, stopping resolution after %d rounds. Unresolved variables may remain.", round+1), // Would be nice to include names of the variables there, but that would complicate things more }) break From 4c3f64afa5b4bfcb42be3599026e6d243402cbad Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 13 Jun 2025 13:27:34 +0200 Subject: [PATCH 2/5] clean up --- .../bundle/variables/issue_3039/bundle/variables/global_vars.yml | 1 - 1 file changed, 1 deletion(-) delete mode 100644 acceptance/bundle/variables/issue_3039/bundle/variables/global_vars.yml diff --git a/acceptance/bundle/variables/issue_3039/bundle/variables/global_vars.yml b/acceptance/bundle/variables/issue_3039/bundle/variables/global_vars.yml deleted file mode 100644 index 8b13789179..0000000000 --- a/acceptance/bundle/variables/issue_3039/bundle/variables/global_vars.yml +++ /dev/null @@ -1 +0,0 @@ - From 340e214c68209397f8cbc46a8a1cf255442f5799 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 13 Jun 2025 13:28:09 +0200 Subject: [PATCH 3/5] clean up --- .../{issue_3039 => issue_3039_lookup_with_ref}/databricks.yml | 0 .../variables/{issue_3039 => issue_3039_lookup_with_ref}/script | 0 .../{issue_3039 => issue_3039_lookup_with_ref}/test.toml | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename acceptance/bundle/variables/{issue_3039 => issue_3039_lookup_with_ref}/databricks.yml (100%) rename acceptance/bundle/variables/{issue_3039 => issue_3039_lookup_with_ref}/script (100%) rename acceptance/bundle/variables/{issue_3039 => issue_3039_lookup_with_ref}/test.toml (100%) diff --git a/acceptance/bundle/variables/issue_3039/databricks.yml b/acceptance/bundle/variables/issue_3039_lookup_with_ref/databricks.yml similarity index 100% rename from acceptance/bundle/variables/issue_3039/databricks.yml rename to acceptance/bundle/variables/issue_3039_lookup_with_ref/databricks.yml diff --git a/acceptance/bundle/variables/issue_3039/script b/acceptance/bundle/variables/issue_3039_lookup_with_ref/script similarity index 100% rename from acceptance/bundle/variables/issue_3039/script rename to acceptance/bundle/variables/issue_3039_lookup_with_ref/script diff --git a/acceptance/bundle/variables/issue_3039/test.toml b/acceptance/bundle/variables/issue_3039_lookup_with_ref/test.toml similarity index 100% rename from acceptance/bundle/variables/issue_3039/test.toml rename to acceptance/bundle/variables/issue_3039_lookup_with_ref/test.toml From 70685a42896befe56a77ea2c06ab59d697819cf4 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 13 Jun 2025 13:29:41 +0200 Subject: [PATCH 4/5] update NEXT_CHANGELOG --- NEXT_CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 186a187268..c6e54dc1fe 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -12,5 +12,6 @@ * Fix reading dashboard contents when the sync root is different than the bundle root ([#3006](https://github.com/databricks/cli/pull/3006)) * When glob for wheels is used, like "\*.whl", it will filter out different version of the same package and will only take the most recent version. ([#2982](https://github.com/databricks/cli/pull/2982)) * When building Python artifacts as part of "bundle deploy" we no longer delete `dist`, `build`, `*egg-info` and `__pycache__` directories. ([#2982](https://github.com/databricks/cli/pull/2982)) +* Fix variable resolution for lookup variables with other references ([#3054](https://github.com/databricks/cli/pull/3054)) ### API Changes From 0be82b606d83b5a0bdfca9f25b0053ba6f1f76bc Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 13 Jun 2025 13:33:12 +0200 Subject: [PATCH 5/5] add missing output.txt --- .../variables/issue_3039_lookup_with_ref/output.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 acceptance/bundle/variables/issue_3039_lookup_with_ref/output.txt diff --git a/acceptance/bundle/variables/issue_3039_lookup_with_ref/output.txt b/acceptance/bundle/variables/issue_3039_lookup_with_ref/output.txt new file mode 100644 index 0000000000..0f85f8131e --- /dev/null +++ b/acceptance/bundle/variables/issue_3039_lookup_with_ref/output.txt @@ -0,0 +1,11 @@ +Error: failed to resolve service-principal: TIDALDBServAccount - usdev, err: ServicePrincipal named 'TIDALDBServAccount - usdev' does not exist + +Name: issue-3039 +Target: personal +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/issue-3039/personal + +Found 1 error + +Exit code: 1