From 1ec84cc3f364dc460f046281a2befd0da62f0779 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 8 Jul 2025 17:56:26 +0200 Subject: [PATCH 1/8] Apply TF fixups to jobs's clusters in initialize phase - This helps to prepare the resource for direct deployment, where we don't do any kind of processing. - This makes configuration changes visible to use in bundle validate/summary. --- .../resourcemutator/job_cluster_fixups.go | 75 +++++++++++++++++++ .../resourcemutator/resource_mutator.go | 3 + 2 files changed, 78 insertions(+) create mode 100644 bundle/config/mutator/resourcemutator/job_cluster_fixups.go diff --git a/bundle/config/mutator/resourcemutator/job_cluster_fixups.go b/bundle/config/mutator/resourcemutator/job_cluster_fixups.go new file mode 100644 index 0000000000..0526cf7842 --- /dev/null +++ b/bundle/config/mutator/resourcemutator/job_cluster_fixups.go @@ -0,0 +1,75 @@ +package resourcemutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/jobs" +) + +type jobClustersFixups struct{} + +func JobClustersFixups() bundle.Mutator { + return &applyTargetMode{} +} + +func (m *jobClustersFixups) Name() string { + return "JobClustersFixups" +} + +func (m *jobClustersFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + for _, job := range b.Config.Resources.Jobs { + if job == nil { + continue + } + prepareJobSettingsForUpdate(&job.JobSettings) + } + + return nil +} + +// Copied from +// https://github.com/databricks/terraform-provider-databricks/blob/a8c92bb/clusters/resource_cluster.go +// https://github.com/databricks/terraform-provider-databricks/blob/a8c92bb130def431b3fadd9fd533c463e8d4813b/clusters/clusters_api.go#L440 +func ModifyRequestOnInstancePool(c *compute.ClusterSpec) { + // Instance profile id does not exist or not set + if c.InstancePoolId == "" { + // Worker must use an instance pool if driver uses an instance pool, + // therefore empty the computed value for driver instance pool. + c.DriverInstancePoolId = "" + return + } + if c.AwsAttributes != nil { + // Reset AwsAttributes + awsAttributes := compute.AwsAttributes{ + InstanceProfileArn: c.AwsAttributes.InstanceProfileArn, + } + c.AwsAttributes = &awsAttributes + } + if c.AzureAttributes != nil { + c.AzureAttributes = &compute.AzureAttributes{} + } + if c.GcpAttributes != nil { + gcpAttributes := compute.GcpAttributes{ + GoogleServiceAccount: c.GcpAttributes.GoogleServiceAccount, + } + c.GcpAttributes = &gcpAttributes + } + c.EnableElasticDisk = false + c.NodeTypeId = "" + c.DriverNodeTypeId = "" +} + +// Copied https://github.com/databricks/terraform-provider-databricks/blob/a8c92bb130def431b3fadd9fd533c463e8d4813b/jobs/resource_job.go#L1016 +func prepareJobSettingsForUpdate(js *jobs.JobSettings) { + for _, task := range js.Tasks { + if task.NewCluster != nil { + ModifyRequestOnInstancePool(task.NewCluster) + } + } + for ind, _ := range js.JobClusters { + ModifyRequestOnInstancePool(&js.JobClusters[ind].NewCluster) + } +} diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index e77c304ca2..a31bb96711 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -158,6 +158,9 @@ func applyNormalizeMutators(ctx context.Context, b *bundle.Bundle) { // Updates (dynamic): resources.dashboards.*.serialized_dashboard // Drops (dynamic): resources.dashboards.*.file_path ConfigureDashboardSerializedDashboard(), + + // Reads and updates (typed): resources.jobs.*.** + JobClustersFixups(), ) } From f738d76d3b9cf4cf5732f11b0c6aeb44a663f5fd Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 8 Jul 2025 17:59:34 +0200 Subject: [PATCH 2/8] enable tests, fmt --- acceptance/bundle/integration_whl/base/test.toml | 1 - acceptance/bundle/integration_whl/custom_params/test.toml | 1 - acceptance/bundle/integration_whl/interactive_cluster/test.toml | 1 - .../bundle/integration_whl/interactive_single_user/test.toml | 1 - acceptance/bundle/integration_whl/wrapper/test.toml | 2 -- .../bundle/integration_whl/wrapper_custom_params/test.toml | 2 -- bundle/config/mutator/resourcemutator/job_cluster_fixups.go | 2 +- 7 files changed, 1 insertion(+), 9 deletions(-) delete mode 100644 acceptance/bundle/integration_whl/base/test.toml delete mode 100644 acceptance/bundle/integration_whl/custom_params/test.toml delete mode 100644 acceptance/bundle/integration_whl/interactive_cluster/test.toml delete mode 100644 acceptance/bundle/integration_whl/interactive_single_user/test.toml diff --git a/acceptance/bundle/integration_whl/base/test.toml b/acceptance/bundle/integration_whl/base/test.toml deleted file mode 100644 index 477c594c43..0000000000 --- a/acceptance/bundle/integration_whl/base/test.toml +++ /dev/null @@ -1 +0,0 @@ -EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"] # Error: deploying jobs.some_other_job: creating: Method=Jobs.Create *retries.Err *apierr.APIError StatusCode=400 ErrorCode="INVALID_PARAMETER_VALUE" Message="The field 'node_type_id' cannot be supplied when an instance pool ID is provided." diff --git a/acceptance/bundle/integration_whl/custom_params/test.toml b/acceptance/bundle/integration_whl/custom_params/test.toml deleted file mode 100644 index 477c594c43..0000000000 --- a/acceptance/bundle/integration_whl/custom_params/test.toml +++ /dev/null @@ -1 +0,0 @@ -EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"] # Error: deploying jobs.some_other_job: creating: Method=Jobs.Create *retries.Err *apierr.APIError StatusCode=400 ErrorCode="INVALID_PARAMETER_VALUE" Message="The field 'node_type_id' cannot be supplied when an instance pool ID is provided." diff --git a/acceptance/bundle/integration_whl/interactive_cluster/test.toml b/acceptance/bundle/integration_whl/interactive_cluster/test.toml deleted file mode 100644 index 477c594c43..0000000000 --- a/acceptance/bundle/integration_whl/interactive_cluster/test.toml +++ /dev/null @@ -1 +0,0 @@ -EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"] # Error: deploying jobs.some_other_job: creating: Method=Jobs.Create *retries.Err *apierr.APIError StatusCode=400 ErrorCode="INVALID_PARAMETER_VALUE" Message="The field 'node_type_id' cannot be supplied when an instance pool ID is provided." diff --git a/acceptance/bundle/integration_whl/interactive_single_user/test.toml b/acceptance/bundle/integration_whl/interactive_single_user/test.toml deleted file mode 100644 index 477c594c43..0000000000 --- a/acceptance/bundle/integration_whl/interactive_single_user/test.toml +++ /dev/null @@ -1 +0,0 @@ -EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"] # Error: deploying jobs.some_other_job: creating: Method=Jobs.Create *retries.Err *apierr.APIError StatusCode=400 ErrorCode="INVALID_PARAMETER_VALUE" Message="The field 'node_type_id' cannot be supplied when an instance pool ID is provided." diff --git a/acceptance/bundle/integration_whl/wrapper/test.toml b/acceptance/bundle/integration_whl/wrapper/test.toml index 1665f5ef7f..6fd2fa8b3b 100644 --- a/acceptance/bundle/integration_whl/wrapper/test.toml +++ b/acceptance/bundle/integration_whl/wrapper/test.toml @@ -1,4 +1,2 @@ # Temporarily disabling due to DBR release breakage. CloudEnvs.gcp = false - -EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"] # Error: deploying jobs.some_other_job: creating: Method=Jobs.Create *retries.Err *apierr.APIError StatusCode=400 ErrorCode="INVALID_PARAMETER_VALUE" Message="The field 'node_type_id' cannot be supplied when an instance pool ID is provided." diff --git a/acceptance/bundle/integration_whl/wrapper_custom_params/test.toml b/acceptance/bundle/integration_whl/wrapper_custom_params/test.toml index 1665f5ef7f..6fd2fa8b3b 100644 --- a/acceptance/bundle/integration_whl/wrapper_custom_params/test.toml +++ b/acceptance/bundle/integration_whl/wrapper_custom_params/test.toml @@ -1,4 +1,2 @@ # Temporarily disabling due to DBR release breakage. CloudEnvs.gcp = false - -EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"] # Error: deploying jobs.some_other_job: creating: Method=Jobs.Create *retries.Err *apierr.APIError StatusCode=400 ErrorCode="INVALID_PARAMETER_VALUE" Message="The field 'node_type_id' cannot be supplied when an instance pool ID is provided." diff --git a/bundle/config/mutator/resourcemutator/job_cluster_fixups.go b/bundle/config/mutator/resourcemutator/job_cluster_fixups.go index 0526cf7842..5c08450510 100644 --- a/bundle/config/mutator/resourcemutator/job_cluster_fixups.go +++ b/bundle/config/mutator/resourcemutator/job_cluster_fixups.go @@ -69,7 +69,7 @@ func prepareJobSettingsForUpdate(js *jobs.JobSettings) { ModifyRequestOnInstancePool(task.NewCluster) } } - for ind, _ := range js.JobClusters { + for ind := range js.JobClusters { ModifyRequestOnInstancePool(&js.JobClusters[ind].NewCluster) } } From cf30853a3f3d66c5b1c97017ee9772965ae8abc4 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 8 Jul 2025 18:02:35 +0200 Subject: [PATCH 3/8] fix --- bundle/config/mutator/resourcemutator/job_cluster_fixups.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/config/mutator/resourcemutator/job_cluster_fixups.go b/bundle/config/mutator/resourcemutator/job_cluster_fixups.go index 5c08450510..709e83bbf6 100644 --- a/bundle/config/mutator/resourcemutator/job_cluster_fixups.go +++ b/bundle/config/mutator/resourcemutator/job_cluster_fixups.go @@ -12,7 +12,7 @@ import ( type jobClustersFixups struct{} func JobClustersFixups() bundle.Mutator { - return &applyTargetMode{} + return &jobClustersFixups{} } func (m *jobClustersFixups) Name() string { From c02364bd93375db8831a5cef5d100ed257244485 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 8 Jul 2025 18:15:15 +0200 Subject: [PATCH 4/8] update out.test files --- acceptance/bundle/integration_whl/custom_params/out.test.toml | 2 +- .../bundle/integration_whl/interactive_cluster/out.test.toml | 2 +- acceptance/bundle/integration_whl/wrapper/out.test.toml | 2 +- .../bundle/integration_whl/wrapper_custom_params/out.test.toml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/integration_whl/custom_params/out.test.toml b/acceptance/bundle/integration_whl/custom_params/out.test.toml index f96be572db..96f495bbd8 100644 --- a/acceptance/bundle/integration_whl/custom_params/out.test.toml +++ b/acceptance/bundle/integration_whl/custom_params/out.test.toml @@ -3,4 +3,4 @@ Cloud = false CloudSlow = true [EnvMatrix] - DATABRICKS_CLI_DEPLOYMENT = ["terraform"] + DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"] diff --git a/acceptance/bundle/integration_whl/interactive_cluster/out.test.toml b/acceptance/bundle/integration_whl/interactive_cluster/out.test.toml index f96be572db..96f495bbd8 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster/out.test.toml +++ b/acceptance/bundle/integration_whl/interactive_cluster/out.test.toml @@ -3,4 +3,4 @@ Cloud = false CloudSlow = true [EnvMatrix] - DATABRICKS_CLI_DEPLOYMENT = ["terraform"] + DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"] diff --git a/acceptance/bundle/integration_whl/wrapper/out.test.toml b/acceptance/bundle/integration_whl/wrapper/out.test.toml index 1bfc8cc307..8bf5f170dd 100644 --- a/acceptance/bundle/integration_whl/wrapper/out.test.toml +++ b/acceptance/bundle/integration_whl/wrapper/out.test.toml @@ -6,4 +6,4 @@ CloudSlow = true gcp = false [EnvMatrix] - DATABRICKS_CLI_DEPLOYMENT = ["terraform"] + DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"] diff --git a/acceptance/bundle/integration_whl/wrapper_custom_params/out.test.toml b/acceptance/bundle/integration_whl/wrapper_custom_params/out.test.toml index 1bfc8cc307..8bf5f170dd 100644 --- a/acceptance/bundle/integration_whl/wrapper_custom_params/out.test.toml +++ b/acceptance/bundle/integration_whl/wrapper_custom_params/out.test.toml @@ -6,4 +6,4 @@ CloudSlow = true gcp = false [EnvMatrix] - DATABRICKS_CLI_DEPLOYMENT = ["terraform"] + DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"] From 691b9d588263c9ded2c741dd902b195569df8fda Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 8 Jul 2025 18:16:01 +0200 Subject: [PATCH 5/8] update --- .../integration_whl/interactive_single_user/out.test.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/integration_whl/interactive_single_user/out.test.toml b/acceptance/bundle/integration_whl/interactive_single_user/out.test.toml index f96be572db..96f495bbd8 100644 --- a/acceptance/bundle/integration_whl/interactive_single_user/out.test.toml +++ b/acceptance/bundle/integration_whl/interactive_single_user/out.test.toml @@ -3,4 +3,4 @@ Cloud = false CloudSlow = true [EnvMatrix] - DATABRICKS_CLI_DEPLOYMENT = ["terraform"] + DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"] From 118f0b8f2dde76b31cddf16fe1a2ad3a0804e7e2 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 9 Jul 2025 09:07:03 +0200 Subject: [PATCH 6/8] enable direct-exp --- acceptance/bundle/integration_whl/base/out.test.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/integration_whl/base/out.test.toml b/acceptance/bundle/integration_whl/base/out.test.toml index f96be572db..96f495bbd8 100644 --- a/acceptance/bundle/integration_whl/base/out.test.toml +++ b/acceptance/bundle/integration_whl/base/out.test.toml @@ -3,4 +3,4 @@ Cloud = false CloudSlow = true [EnvMatrix] - DATABRICKS_CLI_DEPLOYMENT = ["terraform"] + DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"] From 06712d6d37f9b2665d341b4fe6d3191cadb33b5a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 9 Jul 2025 14:36:13 +0200 Subject: [PATCH 7/8] Update bundle/config/mutator/resourcemutator/job_cluster_fixups.go Co-authored-by: Pieter Noordhuis --- bundle/config/mutator/resourcemutator/job_cluster_fixups.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/config/mutator/resourcemutator/job_cluster_fixups.go b/bundle/config/mutator/resourcemutator/job_cluster_fixups.go index 709e83bbf6..27c58b6376 100644 --- a/bundle/config/mutator/resourcemutator/job_cluster_fixups.go +++ b/bundle/config/mutator/resourcemutator/job_cluster_fixups.go @@ -32,7 +32,7 @@ func (m *jobClustersFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Di // Copied from // https://github.com/databricks/terraform-provider-databricks/blob/a8c92bb/clusters/resource_cluster.go -// https://github.com/databricks/terraform-provider-databricks/blob/a8c92bb130def431b3fadd9fd533c463e8d4813b/clusters/clusters_api.go#L440 +// https://github.com/databricks/terraform-provider-databricks/blob/a8c92bb/clusters/clusters_api.go#L440 func ModifyRequestOnInstancePool(c *compute.ClusterSpec) { // Instance profile id does not exist or not set if c.InstancePoolId == "" { From 4554189460b6e003d4f65e1a3eabb3f7920176cc Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 9 Jul 2025 15:31:50 +0200 Subject: [PATCH 8/8] add TODO --- bundle/config/mutator/resourcemutator/job_cluster_fixups.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bundle/config/mutator/resourcemutator/job_cluster_fixups.go b/bundle/config/mutator/resourcemutator/job_cluster_fixups.go index 27c58b6376..b08d9a3a32 100644 --- a/bundle/config/mutator/resourcemutator/job_cluster_fixups.go +++ b/bundle/config/mutator/resourcemutator/job_cluster_fixups.go @@ -24,6 +24,9 @@ func (m *jobClustersFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Di if job == nil { continue } + // TODO: we should raise a warning when user specify both InstancePoolId and NodeTypeId since it's illegal. + // Once we remove TF backend and had warning for some time, we can remove this transformation, the backend + // will reject such configs. prepareJobSettingsForUpdate(&job.JobSettings) }