Fix #3039: Properly resolve lookup variables

NEXT_CHANGELOG.md

@@ -12,5 +12,6 @@
 * Fix reading dashboard contents when the sync root is different than the bundle root ([#3006](https://github.com/databricks/cli/pull/3006))
 * When glob for wheels is used, like "\*.whl", it will filter out different version of the same package and will only take the most recent version. ([#2982](https://github.com/databricks/cli/pull/2982))
 * When building Python artifacts as part of "bundle deploy" we no longer delete `dist`, `build`, `*egg-info` and `__pycache__` directories. ([#2982](https://github.com/databricks/cli/pull/2982))
+* Fix variable resolution for lookup variables with other references ([#3054](https://github.com/databricks/cli/pull/3054))
 
 ### API Changes

acceptance/bundle/variables/complex-cycle-self/output.txt

@@ -1,4 +1,4 @@
-Warning: Detected unresolved variables after 11 resolution rounds
+Warning: Variables references are too deep, stopping resolution after 11 rounds. Unresolved variables may remain.
 
 Name: cycle
 Target: default

acceptance/bundle/variables/complex-cycle/output.txt

@@ -1,4 +1,4 @@
-Warning: Detected unresolved variables after 11 resolution rounds
+Warning: Variables references are too deep, stopping resolution after 11 rounds. Unresolved variables may remain.
 
 Name: cycle
 Target: default

acceptance/bundle/variables/issue_3039_lookup_with_ref/databricks.yml

@@ -0,0 +1,28 @@
+bundle:
+  name: issue-3039
+
+variables:
+  tidal_service_account:
+    description: Gets tidal service account name/id.
+    lookup:
+      service_principal: "TIDALDBServAccount - ${var.uc_catalog}"
+  uc_catalog:
+    description: Unity Catalog prefix.
+    type: string
+    default: ""
+
+non_production_job_permissions: &non_prod_job_permissions
+  permissions:
+    - level: CAN_VIEW
+      service_principal_name: ${var.tidal_service_account}
+
+targets:
+  personal:
+    resources:
+      jobs:
+        xxx_job:
+          <<: *non_prod_job_permissions
+    variables:
+      uc_catalog:
+        description: Unity Catalog prefix.
+        default: "usdev"

acceptance/bundle/variables/issue_3039_lookup_with_ref/output.txt

@@ -0,0 +1,11 @@
+Error: failed to resolve service-principal: TIDALDBServAccount - usdev, err: ServicePrincipal named 'TIDALDBServAccount - usdev' does not exist
+
+Name: issue-3039
+Target: personal
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/issue-3039/personal
+
+Found 1 error
+
+Exit code: 1

acceptance/bundle/variables/issue_3039_lookup_with_ref/script

@@ -0,0 +1 @@
+$CLI bundle validate

acceptance/bundle/variables/issue_3039_lookup_with_ref/test.toml

@@ -0,0 +1,23 @@
+[[Server]]
+Pattern = "GET /api/2.0/preview/scim/v2/ServicePrincipals"
+Response.Body = '''{}'''  # this body causes error, but works to check if resolution is done correctly
+
+# For some reason this body causes SDK to loop forever
+#Response.Body = '''{
+#  "Resources": [
+#    {
+#      "displayName": "TIDALDBServAccount - usdev",
+#      "groups": [],
+#      "id": "10000",
+#      "applicationId": "e700887d-8550-4667-8884-dbc94808cfb6",
+#      "schemas": [
+#        "urn:ietf:params:scim:schemas:core:2.0:ServicePrincipal"
+#      ],
+#      "active": true
+#    }
+#],
+#"totalResults": 1,
+#"startIndex": 1,
+#"itemsPerPage": 1,
+#"schemas": [
+#  "urn:ietf:params:scim:api:messages:2.0:ListResponse"]}'''

bundle/config/mutator/resolve_variable_references.go

@@ -66,11 +66,16 @@ func ResolveVariableReferencesWithoutResources(prefixes ...string) bundle.Mutato
 }
 
 func ResolveVariableReferencesInLookup() bundle.Mutator {
-	return &resolveVariableReferences{prefixes: []string{
-		"bundle",
-		"workspace",
-		"variables",
-	}, pattern: dyn.NewPattern(dyn.Key("variables"), dyn.AnyKey(), dyn.Key("lookup")), lookupFn: lookupForVariables}
+	return &resolveVariableReferences{
+		prefixes: []string{
+			"bundle",
+			"workspace",
+			"variables",
+		},
+		pattern:     dyn.NewPattern(dyn.Key("variables"), dyn.AnyKey(), dyn.Key("lookup")),
+		lookupFn:    lookupForVariables,
+		extraRounds: maxResolutionRounds - 1,
+	}
 }
 
 func lookup(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) {
@@ -149,7 +154,7 @@ func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle)
 		if round >= maxRounds-1 {
 			diags = diags.Append(diag.Diagnostic{
 				Severity: diag.Warning,
-				Summary:  fmt.Sprintf("Detected unresolved variables after %d resolution rounds", round+1),
+				Summary:  fmt.Sprintf("Variables references are too deep, stopping resolution after %d rounds. Unresolved variables may remain.", round+1),
 				// Would be nice to include names of the variables there, but that would complicate things more
 			})
 			break